From 9621ec81fe4e35e4ea20ac119f9ec02f3359c872 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Fri, 22 Oct 2021 23:37:27 +0200 Subject: [PATCH 1/3] lambda_info - Add retry decorators and paginators --- plugins/modules/lambda_info.py | 54 +++++++++------------------------- 1 file changed, 14 insertions(+), 40 deletions(-) diff --git a/plugins/modules/lambda_info.py b/plugins/modules/lambda_info.py index c95c0218132..04cae251be6 100644 --- a/plugins/modules/lambda_info.py +++ b/plugins/modules/lambda_info.py @@ -58,7 +58,6 @@ - name: List all function community.aws.lambda_info: query: all - max_items: 20 register: output - name: show Lambda information ansible.builtin.debug: @@ -89,6 +88,13 @@ from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry + + +@AWSRetry.jittered_backoff() +def _paginate(client, function, **params): + paginator = client.get_paginator(function) + return paginator.paginate(**params).build_full_result() def fix_return(node): @@ -127,14 +133,8 @@ def alias_details(client, module): function_name = module.params.get('function_name') if function_name: - params = dict() - if module.params.get('max_items'): - params['MaxItems'] = module.params.get('max_items') - - if module.params.get('next_marker'): - params['Marker'] = module.params.get('next_marker') try: - lambda_info.update(aliases=client.list_aliases(FunctionName=function_name, **params)['Aliases']) + lambda_info.update(aliases=_paginate(client, 'list_aliases', FunctionName=function_name)['Aliases']) except is_boto3_error_code('ResourceNotFoundException'): lambda_info.update(aliases=[]) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except @@ -154,9 +154,6 @@ def all_details(client, module): :return dict: """ - if module.params.get('max_items') or module.params.get('next_marker'): - module.fail_json(msg='Cannot specify max_items nor next_marker for query=all.') - lambda_info = dict() function_name = module.params.get('function_name') @@ -187,21 +184,14 @@ def config_details(client, module): function_name = module.params.get('function_name') if function_name: try: - lambda_info.update(client.get_function_configuration(FunctionName=function_name)) + lambda_info.update(client.get_function_configuration(aws_retry=True, FunctionName=function_name)) except is_boto3_error_code('ResourceNotFoundException'): lambda_info.update(function={}) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Trying to get {0} configuration".format(function_name)) else: - params = dict() - if module.params.get('max_items'): - params['MaxItems'] = module.params.get('max_items') - - if module.params.get('next_marker'): - params['Marker'] = module.params.get('next_marker') - try: - lambda_info.update(function_list=client.list_functions(**params)['Functions']) + lambda_info.update(function_list=_paginate(client, 'list_functions')['Functions']) except is_boto3_error_code('ResourceNotFoundException'): lambda_info.update(function_list=[]) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except @@ -234,14 +224,8 @@ def mapping_details(client, module): if module.params.get('event_source_arn'): params['EventSourceArn'] = module.params.get('event_source_arn') - if module.params.get('max_items'): - params['MaxItems'] = module.params.get('max_items') - - if module.params.get('next_marker'): - params['Marker'] = module.params.get('next_marker') - try: - lambda_info.update(mappings=client.list_event_source_mappings(**params)['EventSourceMappings']) + lambda_info.update(mappings=_paginate(client, 'list_event_source_mappings', **params)['EventSourceMappings']) except is_boto3_error_code('ResourceNotFoundException'): lambda_info.update(mappings=[]) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except @@ -262,16 +246,13 @@ def policy_details(client, module): :return dict: """ - if module.params.get('max_items') or module.params.get('next_marker'): - module.fail_json(msg='Cannot specify max_items nor next_marker for query=policy.') - lambda_info = dict() function_name = module.params.get('function_name') if function_name: try: # get_policy returns a JSON string so must convert to dict before reassigning to its key - lambda_info.update(policy=json.loads(client.get_policy(FunctionName=function_name)['Policy'])) + lambda_info.update(policy=json.loads(client.get_policy(aws_retry=True, FunctionName=function_name)['Policy'])) except is_boto3_error_code('ResourceNotFoundException'): lambda_info.update(policy={}) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except @@ -295,15 +276,8 @@ def version_details(client, module): function_name = module.params.get('function_name') if function_name: - params = dict() - if module.params.get('max_items'): - params['MaxItems'] = module.params.get('max_items') - - if module.params.get('next_marker'): - params['Marker'] = module.params.get('next_marker') - try: - lambda_info.update(versions=client.list_versions_by_function(FunctionName=function_name, **params)['Versions']) + lambda_info.update(versions=_paginate(client, 'list_versions_by_function', FunctionName=function_name)['Versions']) except is_boto3_error_code('ResourceNotFoundException'): lambda_info.update(versions=[]) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except @@ -343,7 +317,7 @@ def main(): if len(function_name) > 64: module.fail_json(msg='Function name "{0}" exceeds 64 character limit'.format(function_name)) - client = module.client('lambda') + client = module.client('lambda', retry_decorator=AWSRetry.jittered_backoff()) invocations = dict( aliases='alias_details', From 424da908688989b886b5c0cece5d084878356dd9 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Fri, 22 Oct 2021 23:50:42 +0200 Subject: [PATCH 2/3] AWS no longer supports python 2.7 --- .../integration/targets/lambda/tasks/main.yml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/integration/targets/lambda/tasks/main.yml b/tests/integration/targets/lambda/tasks/main.yml index 0013aec25e0..ee5d2aded05 100644 --- a/tests/integration/targets/lambda/tasks/main.yml +++ b/tests/integration/targets/lambda/tasks/main.yml @@ -59,7 +59,7 @@ - name: test with no role or handler lambda: name: ansible-testing-fake-should-not-be-created - runtime: python2.7 + runtime: python3.6 register: result ignore_errors: true - name: assert failure when called with no parameters @@ -73,7 +73,7 @@ - name: test state=present with security group but no vpc lambda: name: '{{ lambda_function_name }}' - runtime: 'python2.7' + runtime: 'python3.6' role: '{{ lambda_role_name }}' zip_file: '{{ zip_res.dest }}' handler: '{{ omit }}' @@ -97,7 +97,7 @@ - name: test state=present - upload the lambda lambda: name: '{{ lambda_function_name }}' - runtime: python2.7 + runtime: python3.6 handler: mini_lambda.handler role: '{{ lambda_role_name }}' zip_file: '{{ zip_res.dest }}' @@ -157,7 +157,7 @@ - name: reset config updates for the following tests lambda: name: '{{lambda_function_name}}' - runtime: python2.7 + runtime: python3.6 tracing_mode: PassThrough handler: mini_lambda.handler role: '{{ lambda_role_name }}' @@ -167,7 +167,7 @@ that: - result is not failed - result.changed == True - - result.configuration.runtime == 'python2.7' + - result.configuration.runtime == 'python3.6' - result.configuration.tracing_config.mode == 'PassThrough' # Query the Lambda @@ -181,7 +181,7 @@ that: - lambda_infos_all is not failed - lambda_infos_all.function[lambda_function_name].function_name == lambda_function_name - - lambda_infos_all.function[lambda_function_name].runtime == "python2.7" + - lambda_infos_all.function[lambda_function_name].runtime == "python3.6" - lambda_infos_all.function[lambda_function_name].versions is defined - lambda_infos_all.function[lambda_function_name].aliases is defined - lambda_infos_all.function[lambda_function_name].policy is defined @@ -253,7 +253,7 @@ - name: test state=present with all nullable variables explicitly set to null lambda: name: '{{lambda_function_name}}' - runtime: python2.7 + runtime: python3.6 role: '{{ lambda_role_name }}' zip_file: '{{zip_res.dest}}' handler: mini_lambda.handler @@ -272,7 +272,7 @@ - name: test putting an environment variable changes lambda lambda: name: '{{lambda_function_name}}' - runtime: python2.7 + runtime: python3.6 handler: mini_lambda.handler role: '{{ lambda_role_name }}' zip_file: '{{zip_res.dest}}' @@ -325,7 +325,7 @@ - name: parallel lambda creation 1/4 lambda: name: '{{lambda_function_name}}_1' - runtime: python2.7 + runtime: python3.6 handler: mini_lambda.handler role: '{{ lambda_role_name }}' zip_file: '{{zip_res.dest}}' @@ -334,7 +334,7 @@ - name: parallel lambda creation 2/4 lambda: name: '{{lambda_function_name}}_2' - runtime: python2.7 + runtime: python3.6 handler: mini_lambda.handler role: '{{ lambda_role_name }}' zip_file: '{{zip_res.dest}}' @@ -343,7 +343,7 @@ - name: parallel lambda creation 3/4 lambda: name: '{{lambda_function_name}}_3' - runtime: python2.7 + runtime: python3.6 handler: mini_lambda.handler role: '{{ lambda_role_name }}' zip_file: '{{zip_res.dest}}' @@ -352,7 +352,7 @@ - name: parallel lambda creation 4/4 lambda: name: '{{lambda_function_name}}_4' - runtime: python2.7 + runtime: python3.6 handler: mini_lambda.handler role: '{{ lambda_role_name }}' zip_file: '{{zip_res.dest}}' From b1732afcb223de0fc5a7e9b988eb7b1c8bee8437 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Fri, 22 Oct 2021 23:58:06 +0200 Subject: [PATCH 3/3] changelog --- changelogs/fragments/777-lambda_info-awsretry.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelogs/fragments/777-lambda_info-awsretry.yml diff --git a/changelogs/fragments/777-lambda_info-awsretry.yml b/changelogs/fragments/777-lambda_info-awsretry.yml new file mode 100644 index 00000000000..6b0db3d5263 --- /dev/null +++ b/changelogs/fragments/777-lambda_info-awsretry.yml @@ -0,0 +1,3 @@ +minor_changes: +- lambda_info - add automatic retries for recoverable errors (https://github.com/ansible-collections/community.aws/pull/777). +- lambda_info - use paginator for list queries (https://github.com/ansible-collections/community.aws/pull/777).