aws_ssm add on_missing and on_denied options (#378)

aws_ssm add on_missing and on_denied options

SUMMARY

closes #126

ISSUE TYPE


Feature Pull Request

COMPONENT NAME

aws_ssm
ADDITIONAL INFORMATION



- set_fact:
     param: "{{ lookup('amazon.aws.aws_ssm', 'variable', on_missing='error', on_denied='warn')}}"

Reviewed-by: Jill R <None>

changelogs/fragments/370-aws_ssm-add-on_missing-and-on_denied-option.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - aws_ssm - add "on_missing" and "on_denied" option (https://github.com/ansible-collections/amazon.aws/pull/370).

plugins/lookup/aws_ssm.py

@@ -54,6 +54,26 @@
     description: Indicates whether to return the name only without path if using a parameter hierarchy.
     default: false
     type: boolean
+  on_missing:
+    description:
+        - Action to take if the SSM parameter is missing.
+        - C(error) will raise a fatal error when the SSM parameter is missing.
+        - C(skip) will silently ignore the missing SSM parameter.
+        - C(warn) will skip over the missing SSM parameter but issue a warning.
+    default: error
+    type: string
+    choices: ['error', 'skip', 'warn']
+    version_added: 2.0.0
+  on_denied:
+    description:
+        - Action to take if access to the SSM parameter is denied.
+        - C(error) will raise a fatal error when access to the SSM parameter is denied.
+        - C(skip) will silently ignore the denied SSM parameter.
+        - C(warn) will skip over the denied SSM parameter but issue a warning.
+    default: error
+    type: string
+    choices: ['error', 'skip', 'warn']
+    version_added: 2.0.0
 '''
 
 EXAMPLES = '''
@@ -104,6 +124,11 @@
   debug: msg='Path contains {{ item }}'
   loop: '{{ lookup("aws_ssm", "/demo/", "/demo1/", bypath=True)}}'
 
+- name: lookup ssm parameter and fail if missing
+  debug: msg="{{ lookup('aws_ssm', 'missing-parameter', on_missing="error" ) }}"
+
+- name: lookup ssm parameter warn if access is denied
+  debug: msg="{{ lookup('aws_ssm', 'missing-parameter', on_denied="warn" ) }}"
 '''
 
 try:
@@ -116,9 +141,11 @@
 from ansible.module_utils._text import to_native
 from ansible.plugins.lookup import LookupBase
 from ansible.utils.display import Display
+from ansible.module_utils.six import string_types
 
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import HAS_BOTO3
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict
+from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
 
 display = Display()
 
@@ -146,7 +173,8 @@ def _boto3_conn(region, credentials):
 class LookupModule(LookupBase):
     def run(self, terms, variables=None, boto_profile=None, aws_profile=None,
             aws_secret_key=None, aws_access_key=None, aws_security_token=None, region=None,
-            bypath=False, shortnames=False, recursive=False, decrypt=True):
+            bypath=False, shortnames=False, recursive=False, decrypt=True, on_missing="skip",
+            on_denied="skip"):
         '''
             :arg terms: a list of lookups to run.
                 e.g. ['parameter_name', 'parameter_name_too' ]
@@ -158,14 +186,21 @@ def run(self, terms, variables=None, boto_profile=None, aws_profile=None,
             :kwarg region: AWS region in which to do the lookup
             :kwarg bypath: Set to True to do a lookup of variables under a path
             :kwarg recursive: Set to True to recurse below the path (requires bypath=True)
+            :kwarg on_missing: Action to take if the SSM parameter is missing
+            :kwarg on_denied: Action to take if access to the SSM parameter is denied
             :returns: A list of parameter values or a list of dictionaries if bypath=True.
         '''
 
         if not HAS_BOTO3:
             raise AnsibleError('botocore and boto3 are required for aws_ssm lookup.')
 
+        # validate arguments 'on_missing' and 'on_denied'
+        if on_missing is not None and (not isinstance(on_missing, string_types) or on_missing.lower() not in ['error', 'warn', 'skip']):
+            raise AnsibleError('"on_missing" must be a string and one of "error", "warn" or "skip", not %s' % on_missing)
+        if on_denied is not None and (not isinstance(on_denied, string_types) or on_denied.lower() not in ['error', 'warn', 'skip']):
+            raise AnsibleError('"on_denied" must be a string and one of "error", "warn" or "skip", not %s' % on_denied)
+
         ret = []
-        response = {}
         ssm_dict = {}
 
         credentials = {}
@@ -214,21 +249,26 @@ def run(self, terms, variables=None, boto_profile=None, aws_profile=None,
             # Lookup by parameter name - always returns a list with one or no entry.
         else:
             display.vvv("AWS_ssm name lookup term: %s" % terms)
-            ssm_dict["Names"] = terms
-            try:
-                response = client.get_parameters(**ssm_dict)
-            except botocore.exceptions.ClientError as e:
-                raise AnsibleError("SSM lookup exception: {0}".format(to_native(e)))
-            params = boto3_tag_list_to_ansible_dict(response['Parameters'], tag_name_key_name="Name",
-                                                    tag_value_key_name="Value")
-            for i in terms:
-                if i.split(':', 1)[0] in params:
-                    ret.append(params[i])
-                elif i in response['InvalidParameters']:
-                    ret.append(None)
-                else:
-                    raise AnsibleError("Ansible internal error: aws_ssm lookup failed to understand boto3 return value: {0}".format(str(response)))
-            return ret
-
+            for term in terms:
+                ret.append(self.get_parameter_value(client, ssm_dict, term, on_missing.lower(), on_denied.lower()))
         display.vvvv("AWS_ssm path lookup returning: %s " % str(ret))
         return ret
+
+    def get_parameter_value(self, client, ssm_dict, term, on_missing, on_denied):
+        ssm_dict["Name"] = term
+        try:
+            response = client.get_parameter(**ssm_dict)
+            return response['Parameter']['Value']
+        except is_boto3_error_code('ParameterNotFound'):
+            if on_missing == 'error':
+                raise AnsibleError("Failed to find SSM parameter %s (ResourceNotFound)" % term)
+            elif on_missing == 'warn':
+                self._display.warning('Skipping, did not find SSM parameter %s' % term)
+        except is_boto3_error_code('AccessDeniedException'):  # pylint: disable=duplicate-except
+            if on_denied == 'error':
+                raise AnsibleError("Failed to access SSM parameter %s (AccessDenied)" % term)
+            elif on_denied == 'warn':
+                self._display.warning('Skipping, access denied for SSM parameter %s' % term)
+        except botocore.exceptions.ClientError as e:  # pylint: disable=duplicate-except
+            raise AnsibleError("SSM lookup exception: {0}".format(to_native(e)))
+        return None

tests/unit/plugins/lookup/test_aws_ssm.py

@@ -34,15 +34,12 @@
     pytestmark = pytest.mark.skip("This test requires the boto3 and botocore Python libraries")
 
 simple_variable_success_response = {
-    'Parameters': [
-        {
-            'Name': 'simple_variable',
-            'Type': 'String',
-            'Value': 'simplevalue',
-            'Version': 1
-        }
-    ],
-    'InvalidParameters': [],
+    'Parameter': {
+        'Name': 'simple_variable',
+        'Type': 'String',
+        'Value': 'simplevalue',
+        'Version': 1
+    },
     'ResponseMetadata': {
         'RequestId': '12121212-3434-5656-7878-9a9a9a9a9a9a',
         'HTTPStatusCode': 200,
@@ -62,17 +59,21 @@
     {'Name': '/testpath/won', 'Type': 'String', 'Value': 'simple_value_won', 'Version': 1}
 ]
 
-missing_variable_response = copy(simple_variable_success_response)
-missing_variable_response['Parameters'] = []
-missing_variable_response['InvalidParameters'] = ['missing_variable']
-
-some_missing_variable_response = copy(simple_variable_success_response)
-some_missing_variable_response['Parameters'] = [
-    {'Name': 'simple', 'Type': 'String', 'Value': 'simple_value', 'Version': 1},
-    {'Name': '/testpath/won', 'Type': 'String', 'Value': 'simple_value_won', 'Version': 1}
-]
-some_missing_variable_response['InvalidParameters'] = ['missing_variable']
+simple_response = copy(simple_variable_success_response)
+simple_response['Parameter'] = {
+    'Name': 'simple',
+    'Type': 'String',
+    'Value': 'simple_value',
+    'Version': 1
+}
 
+simple_won_response = copy(simple_variable_success_response)
+simple_won_response['Parameter'] = {
+    'Name': '/testpath/won',
+    'Type': 'String',
+    'Value': 'simple_value_won',
+    'Version': 1
+}
 
 dummy_credentials = {}
 dummy_credentials['boto_profile'] = None
@@ -82,16 +83,41 @@
 dummy_credentials['region'] = 'eu-west-1'
 
 
+def mock_get_parameter(**kwargs):
+    if kwargs.get('Name') == 'simple':
+        return simple_response
+    elif kwargs.get('Name') == '/testpath/won':
+        return simple_won_response
+    elif kwargs.get('Name') == 'missing_variable':
+        warn_response = {'Error': {'Code': 'ParameterNotFound', 'Message': 'Parameter not found'}}
+        operation_name = 'FakeOperation'
+        raise ClientError(warn_response, operation_name)
+    elif kwargs.get('Name') == 'denied_variable':
+        error_response = {'Error': {'Code': 'AccessDeniedException', 'Message': 'Fake Testing Error'}}
+        operation_name = 'FakeOperation'
+        raise ClientError(error_response, operation_name)
+    elif kwargs.get('Name') == 'notfound_variable':
+        error_response = {'Error': {'Code': 'ResourceNotFoundException', 'Message': 'Fake Testing Error'}}
+        operation_name = 'FakeOperation'
+        raise ClientError(error_response, operation_name)
+    else:
+        warn_response = {'Error': {'Code': 'ParameterNotFound', 'Message': 'Parameter not found'}}
+        operation_name = 'FakeOperation'
+        raise ClientError(warn_response, operation_name)
+
+
 def test_lookup_variable(mocker):
     lookup = aws_ssm.LookupModule()
     lookup._load_name = "aws_ssm"
 
     boto3_double = mocker.MagicMock()
-    boto3_double.Session.return_value.client.return_value.get_parameters.return_value = simple_variable_success_response
+    boto3_double.Session.return_value.client.return_value.get_parameter.return_value = simple_variable_success_response
     boto3_client_double = boto3_double.Session.return_value.client
 
     mocker.patch.object(boto3, 'session', boto3_double)
     retval = lookup.run(["simple_variable"], {}, **dummy_credentials)
+    assert(isinstance(retval, list))
+    assert(len(retval) == 1)
     assert(retval[0] == "simplevalue")
     boto3_client_double.assert_called_with('ssm', 'eu-west-1', aws_access_key_id='notakey',
                                            aws_secret_access_key="notasecret", aws_session_token=None)
@@ -127,10 +153,11 @@ def test_return_none_for_missing_variable(mocker):
     lookup._load_name = "aws_ssm"
 
     boto3_double = mocker.MagicMock()
-    boto3_double.Session.return_value.client.return_value.get_parameters.return_value = missing_variable_response
+    boto3_double.Session.return_value.client.return_value.get_parameter.side_effect = mock_get_parameter
 
     mocker.patch.object(boto3, 'session', boto3_double)
     retval = lookup.run(["missing_variable"], {}, **dummy_credentials)
+    assert(isinstance(retval, list))
     assert(retval[0] is None)
 
 
@@ -143,24 +170,145 @@ def test_match_retvals_to_call_params_even_with_some_missing_variables(mocker):
     lookup._load_name = "aws_ssm"
 
     boto3_double = mocker.MagicMock()
-    boto3_double.Session.return_value.client.return_value.get_parameters.return_value = some_missing_variable_response
+
+    boto3_double.Session.return_value.client.return_value.get_parameter.side_effect = mock_get_parameter
 
     mocker.patch.object(boto3, 'session', boto3_double)
     retval = lookup.run(["simple", "missing_variable", "/testpath/won", "simple"], {}, **dummy_credentials)
+    assert(isinstance(retval, list))
     assert(retval == ["simple_value", None, "simple_value_won", "simple_value"])
 
 
-error_response = {'Error': {'Code': 'ResourceNotFoundException', 'Message': 'Fake Testing Error'}}
-operation_name = 'FakeOperation'
+def test_warn_notfound_resource(mocker):
+    lookup = aws_ssm.LookupModule()
+    lookup._load_name = "aws_ssm"
+
+    boto3_double = mocker.MagicMock()
+    boto3_double.Session.return_value.client.return_value.get_parameter.side_effect = mock_get_parameter
+
+    with pytest.raises(AnsibleError):
+        mocker.patch.object(boto3, 'session', boto3_double)
+        lookup.run(["notfound_variable"], {}, **dummy_credentials)
 
 
-def test_warn_denied_variable(mocker):
+def test_on_missing_wrong_value(mocker):
     lookup = aws_ssm.LookupModule()
     lookup._load_name = "aws_ssm"
 
     boto3_double = mocker.MagicMock()
-    boto3_double.Session.return_value.client.return_value.get_parameters.side_effect = ClientError(error_response, operation_name)
+    boto3_double.Session.return_value.client.return_value.get_parameter.side_effect = mock_get_parameter
 
-    with pytest.raises(AnsibleError):
+    with pytest.raises(AnsibleError) as exc:
+        missing_credentials = copy(dummy_credentials)
+        missing_credentials['on_missing'] = "fake_value_on_missing"
+        mocker.patch.object(boto3, 'session', boto3_double)
+        lookup.run(["simple"], {}, **missing_credentials)
+
+    assert exc.match('"on_missing" must be a string and one of "error", "warn" or "skip"')
+
+
+def test_error_on_missing_variable(mocker):
+    lookup = aws_ssm.LookupModule()
+    lookup._load_name = "aws_ssm"
+
+    boto3_double = mocker.MagicMock()
+    boto3_double.Session.return_value.client.return_value.get_parameter.side_effect = mock_get_parameter
+
+    with pytest.raises(AnsibleError) as exc:
+        missing_credentials = copy(dummy_credentials)
+        missing_credentials['on_missing'] = "error"
         mocker.patch.object(boto3, 'session', boto3_double)
-        lookup.run(["denied_variable"], {}, **dummy_credentials)
+        lookup.run(["missing_variable"], {}, **missing_credentials)
+
+    assert exc.match(r"Failed to find SSM parameter missing_variable \(ResourceNotFound\)")
+
+
+def test_warn_on_missing_variable(mocker):
+    lookup = aws_ssm.LookupModule()
+    lookup._load_name = "aws_ssm"
+
+    boto3_double = mocker.MagicMock()
+    boto3_double.Session.return_value.client.return_value.get_parameter.side_effect = mock_get_parameter
+
+    missing_credentials = copy(dummy_credentials)
+    missing_credentials['on_missing'] = "warn"
+    mocker.patch.object(boto3, 'session', boto3_double)
+    retval = lookup.run(["missing_variable"], {}, **missing_credentials)
+    assert(isinstance(retval, list))
+    assert(retval[0] is None)
+
+
+def test_skip_on_missing_variable(mocker):
+    lookup = aws_ssm.LookupModule()
+    lookup._load_name = "aws_ssm"
+
+    boto3_double = mocker.MagicMock()
+    boto3_double.Session.return_value.client.return_value.get_parameter.side_effect = mock_get_parameter
+
+    missing_credentials = copy(dummy_credentials)
+    missing_credentials['on_missing'] = "warn"
+    mocker.patch.object(boto3, 'session', boto3_double)
+    retval = lookup.run(["missing_variable"], {}, **missing_credentials)
+    assert(isinstance(retval, list))
+    assert(retval[0] is None)
+
+
+def test_on_denied_wrong_value(mocker):
+    lookup = aws_ssm.LookupModule()
+    lookup._load_name = "aws_ssm"
+
+    boto3_double = mocker.MagicMock()
+    boto3_double.Session.return_value.client.return_value.get_parameter.side_effect = mock_get_parameter
+
+    with pytest.raises(AnsibleError) as exc:
+        denied_credentials = copy(dummy_credentials)
+        denied_credentials['on_denied'] = "fake_value_on_denied"
+        mocker.patch.object(boto3, 'session', boto3_double)
+        lookup.run(["simple"], {}, **denied_credentials)
+
+    assert exc.match('"on_denied" must be a string and one of "error", "warn" or "skip"')
+
+
+def test_error_on_denied_variable(mocker):
+    lookup = aws_ssm.LookupModule()
+    lookup._load_name = "aws_ssm"
+
+    boto3_double = mocker.MagicMock()
+    boto3_double.Session.return_value.client.return_value.get_parameter.side_effect = mock_get_parameter
+
+    with pytest.raises(AnsibleError) as exc:
+        denied_credentials = copy(dummy_credentials)
+        denied_credentials['on_denied'] = "error"
+        mocker.patch.object(boto3, 'session', boto3_double)
+        lookup.run(["denied_variable"], {}, **denied_credentials)
+    assert exc.match(r"Failed to access SSM parameter denied_variable \(AccessDenied\)")
+
+
+def test_warn_on_denied_variable(mocker):
+    lookup = aws_ssm.LookupModule()
+    lookup._load_name = "aws_ssm"
+
+    boto3_double = mocker.MagicMock()
+    boto3_double.Session.return_value.client.return_value.get_parameter.side_effect = mock_get_parameter
+
+    denied_credentials = copy(dummy_credentials)
+    denied_credentials['on_denied'] = "warn"
+    mocker.patch.object(boto3, 'session', boto3_double)
+    retval = lookup.run(["denied_variable"], {}, **denied_credentials)
+    assert(isinstance(retval, list))
+    assert(retval[0] is None)
+
+
+def test_skip_on_denied_variable(mocker):
+    lookup = aws_ssm.LookupModule()
+    lookup._load_name = "aws_ssm"
+
+    boto3_double = mocker.MagicMock()
+    boto3_double.Session.return_value.client.return_value.get_parameter.side_effect = mock_get_parameter
+
+    denied_credentials = copy(dummy_credentials)
+    denied_credentials['on_denied'] = "warn"
+    mocker.patch.object(boto3, 'session', boto3_double)
+    retval = lookup.run(["denied_variable"], {}, **denied_credentials)
+    assert(isinstance(retval, list))
+    assert(retval[0] is None)