diff --git a/changelogs/fragments/192-aws_secret-bypath-option.yaml b/changelogs/fragments/192-aws_secret-bypath-option.yaml new file mode 100644 index 00000000000..14e1c25eab2 --- /dev/null +++ b/changelogs/fragments/192-aws_secret-bypath-option.yaml @@ -0,0 +1,2 @@ +minor_changes: +- aws_secret - add ``bypath`` functionality (https://github.com/ansible-collections/amazon.aws/pull/192). diff --git a/plugins/lookup/aws_secret.py b/plugins/lookup/aws_secret.py index 1bc1f1968f7..f5884ee1f9f 100644 --- a/plugins/lookup/aws_secret.py +++ b/plugins/lookup/aws_secret.py @@ -19,12 +19,17 @@ description: - Look up secrets stored in AWS Secrets Manager provided the caller has the appropriate permissions to read the secret. - - Lookup is based on the secret's `Name` value. - - Optional parameters can be passed into this lookup; `version_id` and `version_stage` + - Lookup is based on the secret's I(Name) value. + - Optional parameters can be passed into this lookup; I(version_id) and I(version_stage) options: _terms: description: Name of the secret to look up in AWS Secrets Manager. required: True + bypath: + description: A boolean to indicate whether the parameter is provided as a hierarchy. + default: false + type: boolean + version_added: 1.4.0 version_id: description: Version of the secret(s). required: False @@ -35,6 +40,7 @@ description: - Join two or more entries to form an extended secret. - This is useful for overcoming the 4096 character limit imposed by AWS. + - No effect when used with I(bypath). type: boolean default: false on_missing: @@ -58,6 +64,9 @@ ''' EXAMPLES = r""" + - name: lookup secretsmanager secret in the current region + debug: msg="{{ lookup('amazon.aws.aws_secret', '/path/to/secrets', bypath=true) }}" + - name: Create RDS instance with aws_secret lookup for password param rds: command: create @@ -66,15 +75,15 @@ size: 10 instance_type: db.m1.small username: dbadmin - password: "{{ lookup('aws_secret', 'DbSecret') }}" + password: "{{ lookup('amazon.aws.aws_secret', 'DbSecret') }}" tags: Environment: staging - name: skip if secret does not exist - debug: msg="{{ lookup('aws_secret', 'secret-not-exist', on_missing='skip')}}" + debug: msg="{{ lookup('amazon.aws.aws_secret', 'secret-not-exist', on_missing='skip')}}" - name: warn if access to the secret is denied - debug: msg="{{ lookup('aws_secret', 'secret-denied', on_denied='warn')}}" + debug: msg="{{ lookup('amazon.aws.aws_secret', 'secret-denied', on_denied='warn')}}" """ RETURN = r""" @@ -95,6 +104,7 @@ from ansible.plugins.lookup import LookupBase from ansible.module_utils._text import to_native from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import HAS_BOTO3 def _boto3_conn(region, credentials): @@ -114,70 +124,112 @@ def _boto3_conn(region, credentials): class LookupModule(LookupBase): - def _get_credentials(self): + def run(self, terms, variables=None, boto_profile=None, aws_profile=None, + aws_secret_key=None, aws_access_key=None, aws_security_token=None, region=None, + bypath=False, join=False, version_stage=None, version_id=None, on_missing='error', + on_denied='error'): + ''' + :arg terms: a list of lookups to run. + e.g. ['parameter_name', 'parameter_name_too' ] + :kwarg variables: ansible variables active at the time of the lookup + :kwarg aws_secret_key: identity of the AWS key to use + :kwarg aws_access_key: AWS secret key (matching identity) + :kwarg aws_security_token: AWS session key if using STS + :kwarg decrypt: Set to True to get decrypted parameters + :kwarg region: AWS region in which to do the lookup + :kwarg bypath: Set to True to do a lookup of variables under a path + :kwarg join: Join two or more entries to form an extended secret + :kwarg version_stage: Stage of the secret version + :kwarg version_id: Version of the secret(s) + :kwarg on_missing: Action to take if the secret is missing + :kwarg on_denied: Action to take if access to the secret is denied + :returns: A list of parameter values or a list of dictionaries if bypath=True. + ''' + if not HAS_BOTO3: + raise AnsibleError('botocore and boto3 are required for aws_ssm lookup.') + + missing = on_missing.lower() + if not isinstance(missing, string_types) or missing not in ['error', 'warn', 'skip']: + raise AnsibleError('"on_missing" must be a string and one of "error", "warn" or "skip", not %s' % missing) + + denied = on_denied.lower() + if not isinstance(denied, string_types) or denied not in ['error', 'warn', 'skip']: + raise AnsibleError('"on_denied" must be a string and one of "error", "warn" or "skip", not %s' % denied) + credentials = {} - credentials['aws_profile'] = self.get_option('aws_profile') - credentials['aws_secret_access_key'] = self.get_option('aws_secret_key') - credentials['aws_access_key_id'] = self.get_option('aws_access_key') - credentials['aws_session_token'] = self.get_option('aws_security_token') + if aws_profile: + credentials['aws_profile'] = aws_profile + else: + credentials['aws_profile'] = boto_profile + credentials['aws_secret_access_key'] = aws_secret_key + credentials['aws_access_key_id'] = aws_access_key + credentials['aws_session_token'] = aws_security_token # fallback to IAM role credentials - if not credentials['aws_profile'] and not (credentials['aws_access_key_id'] and credentials['aws_secret_access_key']): + if not credentials['aws_profile'] and not ( + credentials['aws_access_key_id'] and credentials['aws_secret_access_key']): session = botocore.session.get_session() if session.get_credentials() is not None: credentials['aws_access_key_id'] = session.get_credentials().access_key credentials['aws_secret_access_key'] = session.get_credentials().secret_key credentials['aws_session_token'] = session.get_credentials().token - return credentials - - def run(self, terms, variables, **kwargs): - - missing = kwargs.get('on_missing', 'error').lower() - if not isinstance(missing, string_types) or missing not in ['error', 'warn', 'skip']: - raise AnsibleError('"on_missing" must be a string and one of "error", "warn" or "skip", not %s' % missing) - - denied = kwargs.get('on_denied', 'error').lower() - if not isinstance(denied, string_types) or denied not in ['error', 'warn', 'skip']: - raise AnsibleError('"on_denied" must be a string and one of "error", "warn" or "skip", not %s' % denied) - - self.set_options(var_options=variables, direct=kwargs) - boto_credentials = self._get_credentials() - - region = self.get_option('region') - client = _boto3_conn(region, boto_credentials) - - secrets = [] - for term in terms: - params = {} - params['SecretId'] = term - if kwargs.get('version_id'): - params['VersionId'] = kwargs.get('version_id') - if kwargs.get('version_stage'): - params['VersionStage'] = kwargs.get('version_stage') - - try: - response = client.get_secret_value(**params) - if 'SecretBinary' in response: - secrets.append(response['SecretBinary']) - if 'SecretString' in response: - secrets.append(response['SecretString']) - except is_boto3_error_code('ResourceNotFoundException'): - if missing == 'error': - raise AnsibleError("Failed to find secret %s (ResourceNotFound)" % term) - elif missing == 'warn': - self._display.warning('Skipping, did not find secret %s' % term) - except is_boto3_error_code('AccessDeniedException'): # pylint: disable=duplicate-except - if denied == 'error': - raise AnsibleError("Failed to access secret %s (AccessDenied)" % term) - elif denied == 'warn': - self._display.warning('Skipping, access denied for secret %s' % term) - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except - raise AnsibleError("Failed to retrieve secret: %s" % to_native(e)) - - if kwargs.get('join'): - joined_secret = [] - joined_secret.append(''.join(secrets)) - return joined_secret + client = _boto3_conn(region, credentials) + + if bypath: + secrets = {} + for term in terms: + try: + response = client.list_secrets(Filters=[{'Key': 'name', 'Values': [term]}]) + + if 'SecretList' in response: + for secret in response['SecretList']: + secrets.update({secret['Name']: self.get_secret_value(secret['Name'], client, + on_missing=missing, + on_denied=denied)}) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + raise AnsibleError("Failed to retrieve secret: %s" % to_native(e)) + secrets = [secrets] else: - return secrets + secrets = [] + for term in terms: + value = self.get_secret_value(term, client, + version_stage=version_stage, version_id=version_id, + on_missing=missing, on_denied=denied) + if value: + secrets.append(value) + if join: + joined_secret = [] + joined_secret.append(''.join(secrets)) + return joined_secret + + return secrets + + def get_secret_value(self, term, client, version_stage=None, version_id=None, on_missing=None, on_denied=None): + params = {} + params['SecretId'] = term + if version_id: + params['VersionId'] = version_id + if version_stage: + params['VersionStage'] = version_stage + + try: + response = client.get_secret_value(**params) + if 'SecretBinary' in response: + return response['SecretBinary'] + if 'SecretString' in response: + return response['SecretString'] + except is_boto3_error_code('ResourceNotFoundException'): + if on_missing == 'error': + raise AnsibleError("Failed to find secret %s (ResourceNotFound)" % term) + elif on_missing == 'warn': + self._display.warning('Skipping, did not find secret %s' % term) + except is_boto3_error_code('AccessDeniedException'): # pylint: disable=duplicate-except + if on_denied == 'error': + raise AnsibleError("Failed to access secret %s (AccessDenied)" % term) + elif on_denied == 'warn': + self._display.warning('Skipping, access denied for secret %s' % term) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except + raise AnsibleError("Failed to retrieve secret: %s" % to_native(e)) + + return None diff --git a/tests/unit/plugins/lookup/test_aws_secret.py b/tests/unit/plugins/lookup/test_aws_secret.py index dd27bbf60af..6abac2f3078 100644 --- a/tests/unit/plugins/lookup/test_aws_secret.py +++ b/tests/unit/plugins/lookup/test_aws_secret.py @@ -17,6 +17,7 @@ # Make coding more python3-ish from __future__ import (absolute_import, division, print_function) + __metaclass__ = type import pytest @@ -26,6 +27,8 @@ from ansible.errors import AnsibleError from ansible.plugins.loader import lookup_loader +from ansible_collections.amazon.aws.plugins.lookup import aws_secret + try: import boto3 from botocore.exceptions import ClientError @@ -44,35 +47,37 @@ def dummy_credentials(): return dummy_credentials +simple_variable_success_response = { + 'Name': 'secret', + 'VersionId': 'cafe8168-e6ce-4e59-8830-5b143faf6c52', + 'SecretString': '{"secret":"simplesecret"}', + 'VersionStages': ['AWSCURRENT'], + 'ResponseMetadata': { + 'RequestId': '21099462-597c-490a-800f-8b7a41e5151c', + 'HTTPStatusCode': 200, + 'HTTPHeaders': { + 'date': 'Thu, 04 Apr 2019 10:43:12 GMT', + 'content-type': 'application/x-amz-json-1.1', + 'content-length': '252', + 'connection': 'keep-alive', + 'x-amzn-requestid': '21099462-597c-490a-800f-8b7a41e5151c' + }, + 'RetryAttempts': 0 + } +} + + def test_lookup_variable(mocker, dummy_credentials): dateutil_tz = pytest.importorskip("dateutil.tz") - simple_variable_success_response = { - 'Name': 'secret', - 'VersionId': 'cafe8168-e6ce-4e59-8830-5b143faf6c52', - 'SecretString': '{"secret":"simplesecret"}', - 'VersionStages': ['AWSCURRENT'], - 'CreatedDate': datetime.datetime(2019, 4, 4, 11, 41, 0, 878000, tzinfo=dateutil_tz.tzlocal()), - 'ResponseMetadata': { - 'RequestId': '21099462-597c-490a-800f-8b7a41e5151c', - 'HTTPStatusCode': 200, - 'HTTPHeaders': { - 'date': 'Thu, 04 Apr 2019 10:43:12 GMT', - 'content-type': 'application/x-amz-json-1.1', - 'content-length': '252', - 'connection': 'keep-alive', - 'x-amzn-requestid': '21099462-597c-490a-800f-8b7a41e5151c' - }, - 'RetryAttempts': 0 - } - } lookup = lookup_loader.get('amazon.aws.aws_secret') boto3_double = mocker.MagicMock() - boto3_double.Session.return_value.client.return_value.get_secret_value.return_value = simple_variable_success_response + boto3_double.Session.return_value.client.return_value.get_secret_value.return_value = copy( + simple_variable_success_response) boto3_client_double = boto3_double.Session.return_value.client mocker.patch.object(boto3, 'session', boto3_double) retval = lookup.run(["simple_variable"], None, **dummy_credentials) - assert(retval[0] == '{"secret":"simplesecret"}') + assert (retval[0] == '{"secret":"simplesecret"}') boto3_client_double.assert_called_with('secretsmanager', 'eu-west-1', aws_access_key_id='notakey', aws_secret_access_key="notasecret", aws_session_token=None) @@ -122,3 +127,58 @@ def test_on_denied_option(mocker, dummy_credentials): args["on_denied"] = 'warn' retval = lookup_loader.get('amazon.aws.aws_secret').run(["denied_secret"], None, **args) assert(retval == []) + + +def test_path_lookup_variable(mocker, dummy_credentials): + lookup = aws_secret.LookupModule() + lookup._load_name = "aws_secret" + + path_list_secrets_success_response = { + 'SecretList': [ + { + 'Name': '/testpath/too', + }, + { + 'Name': '/testpath/won', + } + ], + 'ResponseMetadata': { + 'RequestId': '21099462-597c-490a-800f-8b7a41e5151c', + 'HTTPStatusCode': 200, + 'HTTPHeaders': { + 'date': 'Thu, 04 Apr 2019 10:43:12 GMT', + 'content-type': 'application/x-amz-json-1.1', + 'content-length': '252', + 'connection': 'keep-alive', + 'x-amzn-requestid': '21099462-597c-490a-800f-8b7a41e5151c' + }, + 'RetryAttempts': 0 + } + } + + boto3_double = mocker.MagicMock() + list_secrets_fn = boto3_double.Session.return_value.client.return_value.list_secrets + list_secrets_fn.return_value = path_list_secrets_success_response + + get_secret_value_fn = boto3_double.Session.return_value.client.return_value.get_secret_value + first_path = copy(simple_variable_success_response) + first_path['SecretString'] = 'simple_value_too' + second_path = copy(simple_variable_success_response) + second_path['SecretString'] = 'simple_value_won' + get_secret_value_fn.side_effect = [ + first_path, + second_path + ] + + boto3_client_double = boto3_double.Session.return_value.client + + mocker.patch.object(boto3, 'session', boto3_double) + dummy_credentials["bypath"] = 'true' + dummy_credentials["boto_profile"] = 'test' + dummy_credentials["aws_profile"] = 'test' + retval = lookup.run(["/testpath"], {}, **dummy_credentials) + assert (retval[0]["/testpath/won"] == "simple_value_won") + assert (retval[0]["/testpath/too"] == "simple_value_too") + boto3_client_double.assert_called_with('secretsmanager', 'eu-west-1', aws_access_key_id='notakey', + aws_secret_access_key="notasecret", aws_session_token=None) + list_secrets_fn.assert_called_with(Filters=[{'Key': 'name', 'Values': ['/testpath']}])