From d948a4e0b29fafdbc4c7628551831cd0a4a5b842 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Tue, 1 Mar 2022 08:17:42 -0800 Subject: [PATCH] aws_secret: Handle pagination for bypath=true (#591) aws_secret: Handle pagination for bypath=true Depends-On: ansible/ansible-zuul-jobs#1359 Depends-On: ansible/ansible-zuul-jobs#1362 Depends-On: ansible/ansible-zuul-jobs#1364 SUMMARY Currently aws_secret returns only 10 secrets, added manual pagination for getting all the secrets as per the task. Fixes #472. ISSUE TYPE Bugfix Pull Request COMPONENT NAME aws_secret Reviewed-by: Alina Buzachis Reviewed-by: Joseph Torcasso (cherry picked from commit 9f0a616e84bd769358509bed6708dd88d7b36179) --- .../591-aws_secrets-handle-pagination.yml | 2 + plugins/lookup/aws_secret.py | 16 ++-- tests/unit/plugins/lookup/test_aws_secret.py | 74 ++++++++++++++++++- 3 files changed, 83 insertions(+), 9 deletions(-) create mode 100644 changelogs/fragments/591-aws_secrets-handle-pagination.yml diff --git a/changelogs/fragments/591-aws_secrets-handle-pagination.yml b/changelogs/fragments/591-aws_secrets-handle-pagination.yml new file mode 100644 index 00000000000..a2df4df7b52 --- /dev/null +++ b/changelogs/fragments/591-aws_secrets-handle-pagination.yml @@ -0,0 +1,2 @@ +minor_changes: +- aws_secret - add pagination for ``bypath`` functionality (https://github.com/ansible-collections/amazon.aws/pull/591). diff --git a/plugins/lookup/aws_secret.py b/plugins/lookup/aws_secret.py index 5eac1a38906..d94fb6256ea 100644 --- a/plugins/lookup/aws_secret.py +++ b/plugins/lookup/aws_secret.py @@ -218,16 +218,18 @@ def run(self, terms, variables=None, boto_profile=None, aws_profile=None, secrets = {} for term in terms: try: - response = client.list_secrets(Filters=[{'Key': 'name', 'Values': [term]}]) + paginator = client.get_paginator('list_secrets') + paginator_response = paginator.paginate( + Filters=[{'Key': 'name', 'Values': [term]}]) + for object in paginator_response: + if 'SecretList' in object: + for secret_obj in object['SecretList']: + secrets.update({secret_obj['Name']: self.get_secret_value( + secret_obj['Name'], client, on_missing=missing, on_denied=denied)}) + secrets = [secrets] - if 'SecretList' in response: - for secret in response['SecretList']: - secrets.update({secret['Name']: self.get_secret_value(secret['Name'], client, - on_missing=missing, - on_denied=denied)}) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: raise AnsibleError("Failed to retrieve secret: %s" % to_native(e)) - secrets = [secrets] else: secrets = [] for term in terms: diff --git a/tests/unit/plugins/lookup/test_aws_secret.py b/tests/unit/plugins/lookup/test_aws_secret.py index 3ac92824864..bed1b354b20 100644 --- a/tests/unit/plugins/lookup/test_aws_secret.py +++ b/tests/unit/plugins/lookup/test_aws_secret.py @@ -163,7 +163,7 @@ def test_nested_lookup_variable(mocker, dummy_credentials): aws_secret_access_key="notasecret", aws_session_token=None) -def test_path_lookup_variable(mocker, dummy_credentials): +def test_path_lookup_variable(mocker, dummy_credentials, record_property): lookup = aws_secret.LookupModule() lookup._load_name = "aws_secret" @@ -205,14 +205,84 @@ def test_path_lookup_variable(mocker, dummy_credentials): ] boto3_client_double = boto3_double.Session.return_value.client + boto3_client_get_paginator_double = boto3_double.Session.return_value.client.return_value.get_paginator + boto3_paginate_double = boto3_client_get_paginator_double.return_value.paginate + boto3_paginate_double.return_value = [path_list_secrets_success_response] mocker.patch.object(boto3, 'session', boto3_double) dummy_credentials["bypath"] = 'true' dummy_credentials["boto_profile"] = 'test' dummy_credentials["aws_profile"] = 'test' retval = lookup.run(["/testpath"], {}, **dummy_credentials) + boto3_paginate_double.assert_called_once() + boto3_client_get_paginator_double.assert_called_once() + boto3_client_get_paginator_double.assert_called_once_with('list_secrets') + + record_property('KEY', retval) assert (retval[0]["/testpath/won"] == "simple_value_won") assert (retval[0]["/testpath/too"] == "simple_value_too") boto3_client_double.assert_called_with('secretsmanager', 'eu-west-1', aws_access_key_id='notakey', aws_secret_access_key="notasecret", aws_session_token=None) - list_secrets_fn.assert_called_with(Filters=[{'Key': 'name', 'Values': ['/testpath']}]) + boto3_paginate_double.assert_called_with(Filters=[{'Key': 'name', 'Values': ['/testpath']}]) + + +def test_path_lookup_variable_paginated(mocker, dummy_credentials, record_property): + lookup = aws_secret.LookupModule() + lookup._load_name = "aws_secret" + + def secret(value): + return {"Name": f"/testpath_paginated/{value}"} + + def get_secret_list(value): + return [secret(f"{value}{i}") for i in range(0, 6)] + path_list_secrets_paginated_success_response = { + 'SecretList': [ + item for pair in zip(get_secret_list("too"), get_secret_list("won")) for item in pair + ], + 'ResponseMetadata': { + 'RequestId': '21099462-597c-490a-800f-8b7a41e5151c', + 'HTTPStatusCode': 200, + 'HTTPHeaders': { + 'date': 'Thu, 04 Apr 2019 10:43:12 GMT', + 'content-type': 'application/x-amz-json-1.1', + 'content-length': '252', + 'connection': 'keep-alive', + 'x-amzn-requestid': '21099462-597c-490a-800f-8b7a41e5151c' + }, + 'RetryAttempts': 0 + } + } + boto3_double = mocker.MagicMock() + list_secrets_fn = boto3_double.Session.return_value.client.return_value.list_secrets + list_secrets_fn.return_value = path_list_secrets_paginated_success_response + get_secret_value_fn = boto3_double.Session.return_value.client.return_value.get_secret_value + + def secret_string(val): + path = copy(simple_variable_success_response) + path["SecretString"] = f"simple_value_{val}" + return path + + def _get_secret_list(value): + return [secret_string(f"{value}{i}") for i in range(0, 6)] + get_secret_value_fn.side_effect = [ + item for pair in zip(_get_secret_list("too"), _get_secret_list("won")) for item in pair + ] + boto3_client_double = boto3_double.Session.return_value.client + boto3_client_get_paginator_double = boto3_double.Session.return_value.client.return_value.get_paginator + boto3_paginate_double = boto3_client_get_paginator_double.return_value.paginate + boto3_paginate_double.return_value = [path_list_secrets_paginated_success_response] + mocker.patch.object(boto3, 'session', boto3_double) + dummy_credentials["bypath"] = 'true' + dummy_credentials["boto_profile"] = 'test' + dummy_credentials["aws_profile"] = 'test' + retval = lookup.run(["/testpath_paginated"], {}, **dummy_credentials) + boto3_paginate_double.assert_called_once() + boto3_client_get_paginator_double.assert_called_once() + boto3_client_get_paginator_double.assert_called_once_with('list_secrets') + record_property('KEY', retval) + assert (retval[0]["/testpath_paginated/won0"] == "simple_value_won0") + assert (retval[0]["/testpath_paginated/too0"] == "simple_value_too0") + assert (len(retval[0]) == 12) + boto3_client_double.assert_called_with('secretsmanager', 'eu-west-1', aws_access_key_id='notakey', + aws_secret_access_key="notasecret", aws_session_token=None) + boto3_paginate_double.assert_called_with(Filters=[{'Key': 'name', 'Values': ['/testpath_paginated']}])