Fix support envvars for lookup aws ssm (ansible-collections#837)

Fix support envvars for lookup aws ssm

SUMMARY

Took over from ansible-collections#623 to apply reviewer's comments and add a changelog fragment.
Working on support in lookup plugin amazon.aws.aws_ssm for endpoint parameters, as well of tradional environment variable for client configuration (AWS_PROFILE, AWS_ACCESS_KEY_ID, ...).
This should fixes ansible-collections#519
Depends-On: ansible-collections#728
ISSUE TYPE

Feature Pull Request

COMPONENT NAME
aws_ssm
ADDITIONAL INFORMATION
It seems LookupModule cannot benefit from AnsibleAWSModule (it expect several methods/params, I am not familiar enough with Python and Ansible to sort it out) ; so this PR uses boto3_conn and get_aws_connection_info defined in utils.ec2.
This is a simple snippet how it can be used:
- name: Load env
  collections:
  - amazon.aws
  gather_facts: no
  hosts: localhost
  connection: local
  vars:
    my_env: "{{ lookup('amazon.aws.aws_ssm', 'status', endpoint='http://localhost:4566') }}"
  tasks:
    - name: show the env
      debug:
        msg: "{{ my_env }}"

Command line test:
ANSIBLE_COLLECTIONS_PATHS=${ANSIBLE_HOME}/collections ansible-playbook test.yml
Output:
PLAY [Load env] *************************************************************************************************************************************************************************************************************************************************************

TASK [show the env] *********************************************************************************************************************************************************************************************************************************************************
ok: [localhost] => {
    "msg": "INIT"
}

PLAY RECAP ******************************************************************************************************************************************************************************************************************************************************************
localhost                  : ok=1    changed=0    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0

Another test with envars:
AWS_ACCESS_KEY_ID='foo' AWS_ACCESS_SECRET_KEY='bar' AWS_URL='http://localhost:4566' ANSIBLE_COLLECTIONS_PATHS=${ANSIBLE_HOME}/collections ansible-playbook test.yml
where we remove the endpoint from lookup:
- name: Load env
  collections:
  - amazon.aws
  gather_facts: no
  hosts: localhost
  connection: local
  vars:
    my_env: "{{ lookup('amazon.aws.aws_ssm', 'status') }}"
  tasks:
    - name: show the env
      debug:
        msg: "{{ my_env }}"
(provides same output)

Reviewed-by: Mark Chappell <None>
(cherry picked from commit 9639325)

changelogs/fragments/837-aws_ssm-envars.yml

@@ -0,0 +1,5 @@
+minor_changes:
+- aws_ssm - Add support for ``endpoint`` parameter (https://github.com/ansible-collections/amazon.aws/pull/837).
+
+bugfixes:
+- aws_ssm - Fix environment variables for client configuration (e.g., AWS_PROFILE, AWS_ACCESS_KEY_ID) (https://github.com/ansible-collections/amazon.aws/pull/837).

plugins/lookup/aws_ssm.py

@@ -70,6 +70,10 @@
     type: string
     choices: ['error', 'skip', 'warn']
     version_added: 2.0.0
+  endpoint:
+    description: Use a custom endpoint when connecting to SSM service.
+    type: string
+    version_added: 3.3.0
 extends_documentation_fragment:
 - amazon.aws.aws_boto3
 '''
@@ -141,38 +145,20 @@
 from ansible.utils.display import Display
 from ansible.module_utils.six import string_types
 
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_conn
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import get_aws_connection_info
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import HAS_BOTO3
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict
 from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
 
 display = Display()
 
 
-def _boto3_conn(region, credentials):
-    if 'boto_profile' in credentials:
-        boto_profile = credentials.pop('boto_profile')
-    else:
-        boto_profile = None
-
-    try:
-        connection = boto3.session.Session(profile_name=boto_profile).client('ssm', region, **credentials)
-    except (botocore.exceptions.ProfileNotFound, botocore.exceptions.PartialCredentialsError):
-        if boto_profile:
-            try:
-                connection = boto3.session.Session(profile_name=boto_profile).client('ssm', region)
-            # FIXME: we should probably do better passing on of the error information
-            except (botocore.exceptions.ProfileNotFound, botocore.exceptions.PartialCredentialsError):
-                raise AnsibleError("Insufficient credentials found.")
-        else:
-            raise AnsibleError("Insufficient credentials found.")
-    return connection
-
-
 class LookupModule(LookupBase):
     def run(self, terms, variables=None, boto_profile=None, aws_profile=None,
             aws_secret_key=None, aws_access_key=None, aws_security_token=None, region=None,
             bypath=False, shortnames=False, recursive=False, decrypt=True, on_missing="skip",
-            on_denied="skip"):
+            on_denied="skip", endpoint=None):
         '''
             :arg terms: a list of lookups to run.
                 e.g. ['parameter_name', 'parameter_name_too' ]
@@ -186,6 +172,7 @@ def run(self, terms, variables=None, boto_profile=None, aws_profile=None,
             :kwarg recursive: Set to True to recurse below the path (requires bypath=True)
             :kwarg on_missing: Action to take if the SSM parameter is missing
             :kwarg on_denied: Action to take if access to the SSM parameter is denied
+            :kwarg endpoint: Endpoint for SSM client
             :returns: A list of parameter values or a list of dictionaries if bypath=True.
         '''
 
@@ -201,16 +188,36 @@ def run(self, terms, variables=None, boto_profile=None, aws_profile=None,
         ret = []
         ssm_dict = {}
 
-        credentials = {}
+        self.params = variables
+
+        cli_region, cli_endpoint, cli_boto_params = get_aws_connection_info(self, boto3=True)
+
+        if region:
+            cli_region = region
+
+        if endpoint:
+            cli_endpoint = endpoint
+
+        # For backward  compatibility
+        if aws_access_key:
+            cli_boto_params.update({'aws_access_key_id': aws_access_key})
+        if aws_secret_key:
+            cli_boto_params.update({'aws_secret_access_key': aws_secret_key})
+        if aws_security_token:
+            cli_boto_params.update({'aws_session_token': aws_security_token})
+        if boto_profile:
+            cli_boto_params.update({'profile_name': boto_profile})
         if aws_profile:
-            credentials['boto_profile'] = aws_profile
-        else:
-            credentials['boto_profile'] = boto_profile
-        credentials['aws_secret_access_key'] = aws_secret_key
-        credentials['aws_access_key_id'] = aws_access_key
-        credentials['aws_session_token'] = aws_security_token
+            cli_boto_params.update({'profile_name': aws_profile})
+
+        cli_boto_params.update(dict(
+            conn_type='client',
+            resource='ssm',
+            region=cli_region,
+            endpoint=cli_endpoint,
+        ))
 
-        client = _boto3_conn(region, credentials)
+        client = boto3_conn(module=self, **cli_boto_params)
 
         ssm_dict['WithDecryption'] = decrypt
 

tests/unit/plugins/lookup/test_aws_ssm.py

@@ -18,9 +18,11 @@
 
 # Make coding more python3-ish
 from __future__ import (absolute_import, division, print_function)
+
 __metaclass__ = type
 
 import pytest
+from unittest.mock import ANY
 from copy import copy
 
 from ansible.errors import AnsibleError
@@ -119,17 +121,26 @@ def test_lookup_variable(mocker):
     assert(isinstance(retval, list))
     assert(len(retval) == 1)
     assert(retval[0] == "simplevalue")
-    boto3_client_double.assert_called_with('ssm', 'eu-west-1', aws_access_key_id='notakey',
-                                           aws_secret_access_key="notasecret", aws_session_token=None)
+    boto3_client_double.assert_called_with(
+        'ssm',
+        region_name='eu-west-1',
+        aws_access_key_id='notakey',
+        aws_secret_access_key="notasecret",
+        aws_session_token=None,
+        endpoint_url=None,
+        config=ANY,
+        verify=None,
+    )
 
 
 def test_path_lookup_variable(mocker):
     lookup = aws_ssm.LookupModule()
     lookup._load_name = "aws_ssm"
 
     boto3_double = mocker.MagicMock()
-    get_path_fn = boto3_double.Session.return_value.client.return_value.get_parameters_by_path
-    get_path_fn.return_value = path_success_response
+    get_paginator_fn = boto3_double.Session.return_value.client.return_value.get_paginator
+    paginator = get_paginator_fn.return_value
+    paginator.paginate.return_value.build_full_result.return_value = path_success_response
     boto3_client_double = boto3_double.Session.return_value.client
 
     mocker.patch.object(boto3, 'session', boto3_double)
@@ -138,9 +149,19 @@ def test_path_lookup_variable(mocker):
     retval = lookup.run(["/testpath"], {}, **args)
     assert(retval[0]["/testpath/won"] == "simple_value_won")
     assert(retval[0]["/testpath/too"] == "simple_value_too")
-    boto3_client_double.assert_called_with('ssm', 'eu-west-1', aws_access_key_id='notakey',
-                                           aws_secret_access_key="notasecret", aws_session_token=None)
-    get_path_fn.assert_called_with(Path="/testpath", Recursive=False, WithDecryption=True)
+    boto3_client_double.assert_called_with(
+        'ssm',
+        region_name='eu-west-1',
+        aws_access_key_id='notakey',
+        aws_secret_access_key="notasecret",
+        aws_session_token=None,
+        endpoint_url=None,
+        config=ANY,
+        verify=None,
+    )
+    get_paginator_fn.assert_called_with('get_parameters_by_path')
+    paginator.paginate.assert_called_with(Path="/testpath", Recursive=True, WithDecryption=True)
+    paginator.paginate.return_value.build_full_result.assert_called_with()
 
 
 def test_return_none_for_missing_variable(mocker):