diff --git a/plugins/modules/lambda.py b/plugins/modules/lambda.py index 8975163cc30..9cb2e0286cc 100644 --- a/plugins/modules/lambda.py +++ b/plugins/modules/lambda.py @@ -212,21 +212,24 @@ ''' from ansible.module_utils._text import to_native +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import get_aws_connection_info, boto3_conn, camel_dict_to_snake_dict +from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags + import base64 import hashlib import traceback import re try: - from botocore.exceptions import ClientError, BotoCoreError, ValidationError, ParamValidationError + from botocore.exceptions import ClientError, BotoCoreError except ImportError: pass # protected by AnsibleAWSModule -def get_account_info(module, region=None, endpoint=None, **aws_connect_kwargs): +def get_account_info(module): """return the account information (account id and partition) we are currently working on get_account_info tries too find out the account that we are working @@ -237,27 +240,25 @@ def get_account_info(module, region=None, endpoint=None, **aws_connect_kwargs): account_id = None partition = None try: - sts_client = boto3_conn(module, conn_type='client', resource='sts', - region=region, endpoint=endpoint, **aws_connect_kwargs) - caller_id = sts_client.get_caller_identity() + sts_client = module.client('sts', retry_decorator=AWSRetry.jittered_backoff()) + caller_id = sts_client.get_caller_identity(aws_retry=True) account_id = caller_id.get('Account') partition = caller_id.get('Arn').split(':')[1] - except ClientError: + except (BotoCoreError, ClientError): try: - iam_client = boto3_conn(module, conn_type='client', resource='iam', - region=region, endpoint=endpoint, **aws_connect_kwargs) - arn, partition, service, reg, account_id, resource = iam_client.get_user()['User']['Arn'].split(':') - except ClientError as e: - if (e.response['Error']['Code'] == 'AccessDenied'): + iam_client = module.client('iam', retry_decorator=AWSRetry.jittered_backoff()) + arn, partition, service, reg, account_id, resource = iam_client.get_user(aws_retry=True)['User']['Arn'].split(':') + except is_boto3_error_code('AccessDenied') as e: + try: except_msg = to_native(e.message) - m = except_msg.search(r"arn:(aws(-([a-z\-]+))?):iam::([0-9]{12,32}):\w+/") - account_id = m.group(4) - partition = m.group(1) - if account_id is None: + except AttributeError: + except_msg = to_native(e) + m = re.search(r"arn:(aws(-([a-z\-]+))?):iam::([0-9]{12,32}):\w+/", except_msg) + if m is None: module.fail_json_aws(e, msg="getting account information") - if partition is None: - module.fail_json_aws(e, msg="getting account information: partition") - except Exception as e: + account_id = m.group(4) + partition = m.group(1) + except (BotoCoreError, ClientError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="getting account information") return account_id, partition @@ -266,15 +267,10 @@ def get_account_info(module, region=None, endpoint=None, **aws_connect_kwargs): def get_current_function(connection, function_name, qualifier=None): try: if qualifier is not None: - return connection.get_function(FunctionName=function_name, Qualifier=qualifier) - return connection.get_function(FunctionName=function_name) - except ClientError as e: - try: - if e.response['Error']['Code'] == 'ResourceNotFoundException': - return None - except (KeyError, AttributeError): - pass - raise e + return connection.get_function(FunctionName=function_name, Qualifier=qualifier, aws_retry=True) + return connection.get_function(FunctionName=function_name, aws_retry=True) + except is_boto3_error_code('ResourceNotFoundException'): + return None def sha256sum(filename): @@ -290,17 +286,14 @@ def sha256sum(filename): def set_tag(client, module, tags, function): - if not hasattr(client, "list_tags"): - module.fail_json(msg="Using tags requires botocore 1.5.40 or above") changed = False arn = function['Configuration']['FunctionArn'] try: - current_tags = client.list_tags(Resource=arn).get('Tags', {}) - except ClientError as e: - module.fail_json(msg="Unable to list tags: {0}".format(to_native(e)), - exception=traceback.format_exc()) + current_tags = client.list_tags(Resource=arn, aws_retry=True).get('Tags', {}) + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e, msg="Unable to list tags") tags_to_add, tags_to_remove = compare_aws_tags(current_tags, tags, purge_tags=True) @@ -308,24 +301,21 @@ def set_tag(client, module, tags, function): if tags_to_remove: client.untag_resource( Resource=arn, - TagKeys=tags_to_remove + TagKeys=tags_to_remove, + aws_retry=True ) changed = True if tags_to_add: client.tag_resource( Resource=arn, - Tags=tags_to_add + Tags=tags_to_add, + aws_retry=True ) changed = True - except ClientError as e: - module.fail_json(msg="Unable to tag resource {0}: {1}".format(arn, - to_native(e)), exception=traceback.format_exc(), - **camel_dict_to_snake_dict(e.response)) - except BotoCoreError as e: - module.fail_json(msg="Unable to tag resource {0}: {1}".format(arn, - to_native(e)), exception=traceback.format_exc()) + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e, msg="Unable to tag resource {0}".format(arn)) return changed @@ -389,22 +379,21 @@ def main(): check_mode = module.check_mode changed = False - region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True) - if not region: - module.fail_json(msg='region must be specified') - try: - client = boto3_conn(module, conn_type='client', resource='lambda', - region=region, endpoint=ec2_url, **aws_connect_kwargs) - except (ClientError, ValidationError) as e: + client = module.client('lambda', retry_decorator=AWSRetry.jittered_backoff()) + except (ClientError, BotoCoreError) as e: module.fail_json_aws(e, msg="Trying to connect to AWS") + if tags is not None: + if not hasattr(client, "list_tags"): + module.fail_json(msg="Using tags requires botocore 1.5.40 or above") + if state == 'present': if re.match(r'^arn:aws(-([a-z\-]+))?:iam', role): role_arn = role else: # get account ID and assemble ARN - account_id, partition = get_account_info(module, region=region, endpoint=ec2_url, **aws_connect_kwargs) + account_id, partition = get_account_info(module) role_arn = 'arn:{0}:iam::{1}:role/{2}'.format(partition, account_id, role) # Get function configuration if present, False otherwise @@ -447,9 +436,7 @@ def main(): func_kwargs.update({'TracingConfig': {'Mode': tracing_mode}}) # If VPC configuration is desired - if vpc_subnet_ids or vpc_security_group_ids: - if not vpc_subnet_ids or not vpc_security_group_ids: - module.fail_json(msg='vpc connectivity requires at least one security group and one subnet') + if vpc_subnet_ids: if 'VpcConfig' in current_config: # Compare VPC config with current config @@ -472,10 +459,10 @@ def main(): if len(func_kwargs) > 1: try: if not check_mode: - response = client.update_function_configuration(**func_kwargs) + response = client.update_function_configuration(aws_retry=True, **func_kwargs) current_version = response['Version'] changed = True - except (ParamValidationError, ClientError) as e: + except (BotoCoreError, ClientError) as e: module.fail_json_aws(e, msg="Trying to update lambda configuration") # Update code configuration @@ -513,10 +500,10 @@ def main(): if len(code_kwargs) > 2: try: if not check_mode: - response = client.update_function_code(**code_kwargs) + response = client.update_function_code(aws_retry=True, **code_kwargs) current_version = response['Version'] changed = True - except (ParamValidationError, ClientError) as e: + except (BotoCoreError, ClientError) as e: module.fail_json_aws(e, msg="Trying to upload new code") # Describe function code and configuration @@ -573,10 +560,7 @@ def main(): func_kwargs.update({'TracingConfig': {'Mode': tracing_mode}}) # If VPC configuration is given - if vpc_subnet_ids or vpc_security_group_ids: - if not vpc_subnet_ids or not vpc_security_group_ids: - module.fail_json(msg='vpc connectivity requires at least one security group and one subnet') - + if vpc_subnet_ids: func_kwargs.update({'VpcConfig': {'SubnetIds': vpc_subnet_ids, 'SecurityGroupIds': vpc_security_group_ids}}) @@ -584,10 +568,10 @@ def main(): current_version = None try: if not check_mode: - response = client.create_function(**func_kwargs) + response = client.create_function(aws_retry=True, **func_kwargs) current_version = response['Version'] changed = True - except (ParamValidationError, ClientError) as e: + except (BotoCoreError, ClientError) as e: module.fail_json_aws(e, msg="Trying to create function") # Tag Function @@ -604,9 +588,9 @@ def main(): if state == 'absent' and current_function: try: if not check_mode: - client.delete_function(FunctionName=name) + client.delete_function(FunctionName=name, aws_retry=True) changed = True - except (ParamValidationError, ClientError) as e: + except (BotoCoreError, ClientError) as e: module.fail_json_aws(e, msg="Trying to delete Lambda function") module.exit_json(changed=changed) diff --git a/tests/integration/targets/lambda/aliases b/tests/integration/targets/lambda/aliases new file mode 100644 index 00000000000..c11244e5865 --- /dev/null +++ b/tests/integration/targets/lambda/aliases @@ -0,0 +1,4 @@ +cloud/aws +shippable/aws/group2 +execute_lambda +lambda_info diff --git a/tests/integration/targets/lambda/defaults/main.yml b/tests/integration/targets/lambda/defaults/main.yml new file mode 100644 index 00000000000..d227210344f --- /dev/null +++ b/tests/integration/targets/lambda/defaults/main.yml @@ -0,0 +1,3 @@ +--- +# defaults file for aws_lambda test +lambda_function_name: '{{resource_prefix}}' diff --git a/tests/integration/targets/lambda/files/mini_lambda.py b/tests/integration/targets/lambda/files/mini_lambda.py new file mode 100644 index 00000000000..901f6b55a77 --- /dev/null +++ b/tests/integration/targets/lambda/files/mini_lambda.py @@ -0,0 +1,48 @@ +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json +import os + + +def handler(event, context): + """ + The handler function is the function which gets called each time + the lambda is run. + """ + # printing goes to the cloudwatch log allowing us to simply debug the lambda if we can find + # the log entry. + print("got event:\n" + json.dumps(event)) + + # if the name parameter isn't present this can throw an exception + # which will result in an amazon chosen failure from the lambda + # which can be completely fine. + + name = event["name"] + + # we can use environment variables as part of the configuration of the lambda + # which can change the behaviour of the lambda without needing a new upload + + extra = os.environ.get("EXTRA_MESSAGE") + if extra is not None and len(extra) > 0: + greeting = "hello {0}. {1}".format(name, extra) + else: + greeting = "hello " + name + + return {"message": greeting} + + +def main(): + """ + This main function will normally never be called during normal + lambda use. It is here for testing the lambda program only. + """ + event = {"name": "james"} + context = None + print(handler(event, context)) + + +if __name__ == '__main__': + main() diff --git a/tests/integration/targets/lambda/meta/main.yml b/tests/integration/targets/lambda/meta/main.yml new file mode 100644 index 00000000000..1f64f1169a9 --- /dev/null +++ b/tests/integration/targets/lambda/meta/main.yml @@ -0,0 +1,3 @@ +dependencies: + - prepare_tests + - setup_ec2 diff --git a/tests/integration/targets/lambda/tasks/main.yml b/tests/integration/targets/lambda/tasks/main.yml new file mode 100644 index 00000000000..403cdd411f6 --- /dev/null +++ b/tests/integration/targets/lambda/tasks/main.yml @@ -0,0 +1,406 @@ +- name: set connection information for AWS modules and run tests + module_defaults: + group/aws: + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token | default(omit) }}' + region: '{{ aws_region }}' + collections: + - community.general + block: + - name: test with no parameters + lambda: + register: result + ignore_errors: true + - name: assert failure when called with no parameters + assert: + that: + - result.failed + - 'result.msg.startswith("missing required arguments: name")' + - name: test with no parameters except state absent + lambda: + state: absent + register: result + ignore_errors: true + - name: assert failure when called with no parameters + assert: + that: + - result.failed + - 'result.msg.startswith("missing required arguments: name")' + - name: test with no role or handler + lambda: + name: ansible-testing-fake-should-not-be-created + runtime: python2.7 + register: result + ignore_errors: true + - name: assert failure when called with no parameters + assert: + that: + - result.failed + - 'result.msg.startswith("state is present but all of the following are missing: handler")' + - name: test with all module required variables but no region + lambda: + name: ansible-testing-fake-should-not-be-created + runtime: python2.7 + handler: no-handler + role: arn:fake-role-doesnt-exist + region: '{{ omit }}' + register: result + ignore_errors: true + - name: assert failure when called with only 'name' + assert: + that: + - result.failed + - '"requires a region and none was found" in result.msg' + - name: move lambda into place for archive module + copy: + src: mini_lambda.py + dest: '{{output_dir}}/mini_lambda.py' + mode: preserve + - name: bundle lambda into a zip + register: zip_res + archive: + format: zip + path: '{{output_dir}}/mini_lambda.py' + dest: '{{output_dir}}/mini_lambda.zip' + - name: test state=present - upload the lambda + lambda: + name: '{{lambda_function_name}}' + runtime: python2.7 + handler: mini_lambda.handler + role: ansible_lambda_role + zip_file: '{{zip_res.dest}}' + register: result + - name: assert lambda upload succeeded + assert: + that: + - result is not failed + - result.configuration.tracing_config.mode == "PassThrough" + - name: test lambda works + execute_lambda: + name: '{{lambda_function_name}}' + payload: + name: Mr Ansible Tests + register: result + - name: assert lambda manages to respond as expected + assert: + that: + - result is not failed + - result.result.output.message == "hello Mr Ansible Tests" + - name: test lambda config updates + lambda: + name: '{{lambda_function_name}}' + runtime: nodejs10.x + tracing_mode: Active + handler: mini_lambda.handler + role: ansible_lambda_role + register: update_result + - name: assert that update succeeded + assert: + that: + - update_result is not failed + - update_result.changed == True + - update_result.configuration.runtime == 'nodejs10.x' + - update_result.configuration.tracing_config.mode == 'Active' + - name: test no changes are made with the same parameters + lambda: + name: '{{lambda_function_name}}' + runtime: nodejs10.x + tracing_mode: Active + handler: mini_lambda.handler + role: ansible_lambda_role + register: update_result + - name: assert that update succeeded + assert: + that: + - update_result is not failed + - update_result.changed == False + - update_result.configuration.runtime == 'nodejs10.x' + - update_result.configuration.tracing_config.mode == 'Active' + - name: reset config updates for the following tests + lambda: + name: '{{lambda_function_name}}' + runtime: python2.7 + tracing_mode: PassThrough + handler: mini_lambda.handler + role: ansible_lambda_role + register: result + - name: assert that reset succeeded + assert: + that: + - result is not failed + - result.changed == True + - result.configuration.runtime == 'python2.7' + - result.configuration.tracing_config.mode == 'PassThrough' + - name: lambda_info | Gather all infos for given lambda function + lambda_info: + name: '{{ lambda_function_name }}' + query: all + register: lambda_infos_all + - name: lambda_info | Assert successfull retrieval of all information + assert: + that: + - lambda_infos_all is not failed + - lambda_infos_all.function[lambda_function_name].function_name == lambda_function_name + - lambda_infos_all.function[lambda_function_name].runtime == "python2.7" + - lambda_infos_all.function[lambda_function_name].versions is defined + - lambda_infos_all.function[lambda_function_name].aliases is defined + - lambda_infos_all.function[lambda_function_name].policy is defined + - lambda_infos_all.function[lambda_function_name].mappings is defined + - lambda_infos_all.function[lambda_function_name].description == "" + - lambda_infos_all.function[lambda_function_name].function_arn is defined + - lambda_infos_all.function[lambda_function_name].handler == "mini_lambda.handler" + - name: lambda_info | Gather version infos for given lambda function + lambda_info: + name: '{{ lambda_function_name }}' + query: versions + register: lambda_infos_versions + - name: lambda_info | Assert successfull retrieval of versions information + assert: + that: + - lambda_infos_versions is not failed + - lambda_infos_versions.function[lambda_function_name].versions|length > 0 + - lambda_infos_versions.function[lambda_function_name].function_name is undefined + - name: lambda_info | Gather config infos for given lambda function + lambda_info: + name: '{{ lambda_function_name }}' + query: config + register: lambda_infos_config + - name: lambda_info | Assert successfull retrieval of config information + assert: + that: + - lambda_infos_config is not failed + - lambda_infos_config.function[lambda_function_name].function_name == lambda_function_name + - lambda_infos_config.function[lambda_function_name].description is defined + - lambda_infos_config.function[lambda_function_name].versions is undefined + - name: lambda_info | Gather policy infos for given lambda function + lambda_info: + name: '{{ lambda_function_name }}' + query: policy + register: lambda_infos_policy + - name: lambda_info | Assert successfull retrieval of policy information + assert: + that: + - lambda_infos_policy is not failed + - lambda_infos_policy.function[lambda_function_name].policy is defined + - lambda_infos_policy.function[lambda_function_name].versions is undefined + - name: lambda_info | Gather aliases infos for given lambda function + lambda_info: + name: '{{ lambda_function_name }}' + query: aliases + register: lambda_infos_aliases + - name: lambda_info | Assert successfull retrieval of aliases information + assert: + that: + - lambda_infos_aliases is not failed + - lambda_infos_aliases.function[lambda_function_name].aliases is defined + - name: lambda_info | Gather mappings infos for given lambda function + lambda_info: + name: '{{ lambda_function_name }}' + query: mappings + register: lambda_infos_mappings + - name: lambda_info | Assert successfull retrieval of mappings information + assert: + that: + - lambda_infos_mappings is not failed + - lambda_infos_mappings.function[lambda_function_name].mappings is defined + - name: test state=present with security group but no vpc + lambda: + name: '{{lambda_function_name}}' + runtime: python2.7 + role: ansible_lambda_role + zip_file: '{{zip_res.dest}}' + handler: '{{ omit }}' + description: '{{ omit }}' + vpc_subnet_ids: '{{ omit }}' + vpc_security_group_ids: sg-FA6E + environment_variables: '{{ omit }}' + dead_letter_arn: '{{ omit }}' + register: result + ignore_errors: true + - name: assert lambda fails with proper message + assert: + that: + - result is failed + - result.msg != "MODULE FAILURE" + - result.changed == False + - '"parameters are required together" in result.msg' + - name: test state=present with all nullable variables explicitly set to null + lambda: + name: '{{lambda_function_name}}' + runtime: python2.7 + role: ansible_lambda_role + zip_file: '{{zip_res.dest}}' + handler: mini_lambda.handler + description: null + vpc_subnet_ids: null + vpc_security_group_ids: null + environment_variables: null + dead_letter_arn: null + register: result + - name: assert lambda remains as before + assert: + that: + - result is not failed + - result.changed == False + - name: test putting an environment variable changes lambda + lambda: + name: '{{lambda_function_name}}' + runtime: python2.7 + handler: mini_lambda.handler + role: ansible_lambda_role + zip_file: '{{zip_res.dest}}' + environment_variables: + EXTRA_MESSAGE: I think you are great!! + register: result + - name: assert lambda upload succeeded + assert: + that: + - result is not failed + - result.changed == True + - name: test lambda works + execute_lambda: + name: '{{lambda_function_name}}' + payload: + name: Mr Ansible Tests + security_token: '{{security_token}}' + register: result + - name: assert lambda manages to respond as expected + assert: + that: + - result is not failed + - result.result.output.message == "hello Mr Ansible Tests. I think you are great!!" + - name: test state=present triggering a network exception due to bad url + lambda: + name: '{{lambda_function_name}}' + runtime: python2.7 + role: ansible_lambda_role + ec2_url: https://noexist.example.com + ec2_region: '{{ec2_region}}' + ec2_access_key: iamnotreallyanaccesskey + ec2_secret_key: thisisabadsecretkey + security_token: andthisisabadsecuritytoken + zip_file: '{{zip_res.dest}}' + register: result + ignore_errors: true + - name: assert lambda manages to respond as expected + assert: + that: + - result is failed + - result.changed == False + - name: test state=absent (expect changed=False) + lambda: + name: '{{lambda_function_name}}' + state: absent + register: result + - name: assert state=absent + assert: + that: + - result is not failed + - result.changed == True + - name: parallel lambda creation 1/4 + lambda: + name: '{{lambda_function_name}}_1' + runtime: python2.7 + handler: mini_lambda.handler + role: ansible_lambda_role + zip_file: '{{zip_res.dest}}' + async: 1000 + register: async_1 + - name: parallel lambda creation 2/4 + lambda: + name: '{{lambda_function_name}}_2' + runtime: python2.7 + handler: mini_lambda.handler + role: ansible_lambda_role + zip_file: '{{zip_res.dest}}' + async: 1000 + register: async_2 + - name: parallel lambda creation 3/4 + lambda: + name: '{{lambda_function_name}}_3' + runtime: python2.7 + handler: mini_lambda.handler + role: ansible_lambda_role + zip_file: '{{zip_res.dest}}' + async: 1000 + register: async_3 + - name: parallel lambda creation 4/4 + lambda: + name: '{{lambda_function_name}}_4' + runtime: python2.7 + handler: mini_lambda.handler + role: ansible_lambda_role + zip_file: '{{zip_res.dest}}' + register: result + - name: assert lambda manages to respond as expected + assert: + that: + - result is not failed + - name: wait for async job 1 + async_status: jid={{ async_1.ansible_job_id }} + register: job_result + until: job_result is finished + retries: 30 + - name: wait for async job 2 + async_status: jid={{ async_1.ansible_job_id }} + register: job_result + until: job_result is finished + retries: 30 + - name: wait for async job 3 + async_status: jid={{ async_3.ansible_job_id }} + register: job_result + until: job_result is finished + retries: 30 + - name: parallel lambda deletion 1/4 + lambda: + name: '{{lambda_function_name}}_1' + state: absent + zip_file: '{{zip_res.dest}}' + async: 1000 + register: async_1 + - name: parallel lambda deletion 2/4 + lambda: + name: '{{lambda_function_name}}_2' + state: absent + zip_file: '{{zip_res.dest}}' + async: 1000 + register: async_2 + - name: parallel lambda deletion 3/4 + lambda: + name: '{{lambda_function_name}}_3' + state: absent + zip_file: '{{zip_res.dest}}' + async: 1000 + register: async_3 + - name: parallel lambda deletion 4/4 + lambda: + name: '{{lambda_function_name}}_4' + state: absent + zip_file: '{{zip_res.dest}}' + register: result + - name: assert lambda creation has succeeded + assert: + that: + - result is not failed + - name: wait for async job 1 + async_status: jid={{ async_1.ansible_job_id }} + register: job_result + until: job_result is finished + retries: 30 + - name: wait for async job 2 + async_status: jid={{ async_1.ansible_job_id }} + register: job_result + until: job_result is finished + retries: 30 + - name: wait for async job 3 + async_status: jid={{ async_3.ansible_job_id }} + register: job_result + until: job_result is finished + retries: 30 + always: + - name: ensure function is absent at end of test + lambda: + name: '{{lambda_function_name}}' + state: absent + ignore_errors: true