From 6989959b0f8a6cf87bfc25244bd8740717e3da68 Mon Sep 17 00:00:00 2001
From: Mark Chappell <mchappel@redhat.com>
Date: Mon, 23 Nov 2020 23:43:07 +0100
Subject: [PATCH] Remove integration test reliance on the pre-existence of
 ansible_lambda_role (#63)

* Migrate lambda tests to a unique IAM role (and clean up afterwards)

* Migrate lambda_policy to a unique IAM role per-test (and clean up)

* Use the Amazon provided AWSXrayWriteOnlyAccess policy rather than creating something custom

* Migrate sns_topic to a unique IAM role per-test (and clean up)

* Migrate s3_bucket_notifications to a unique IAM role per-test (and clean up)

* CI relies on the very specific ansible-test-* pattern for role names

* Minor lambda test cleanup
- Reorder
- Comments and spacing
- Remove testing for standard AnsibleAWSModule boto behaviour, we have thorough tests for this now

This commit was initially merged in https://github.com/ansible-collections/community.aws
See: https://github.com/ansible-collections/community.aws/commit/60a4758f4d2db1f279819095424db5d9b2d9b9e2
---
 .../targets/lambda/defaults/main.yml          |  11 +-
 .../lambda/files/minimal_trust_policy.json    |  12 ++
 .../integration/targets/lambda/tasks/main.yml | 163 ++++++++++--------
 .../targets/lambda_policy/defaults/main.yml   |  11 +-
 .../targets/lambda_policy/tasks/main.yml      |  77 ++++-----
 5 files changed, 155 insertions(+), 119 deletions(-)
 create mode 100644 tests/integration/targets/lambda/files/minimal_trust_policy.json

diff --git a/tests/integration/targets/lambda/defaults/main.yml b/tests/integration/targets/lambda/defaults/main.yml
index d227210344f..ea29794efe0 100644
--- a/tests/integration/targets/lambda/defaults/main.yml
+++ b/tests/integration/targets/lambda/defaults/main.yml
@@ -1,3 +1,10 @@
 ---
-# defaults file for aws_lambda test
-lambda_function_name: '{{resource_prefix}}'
+# defaults file for lambda integration test
+lambda_function_name: '{{ resource_prefix }}'
+# IAM role names have to be less than 64 characters
+# The 8 digit identifier at the end of resource_prefix helps determine during
+# which test something was created and allows tests to be run in parallel
+# Shippable resource_prefixes are in the format shippable-123456-123, so in those cases
+# we need both sets of digits to keep the resource name unique
+unique_id: "{{ resource_prefix | regex_search('(\\d+-?)(\\d+)$') }}"
+lambda_role_name: 'ansible-test-{{ unique_id }}-lambda'
diff --git a/tests/integration/targets/lambda/files/minimal_trust_policy.json b/tests/integration/targets/lambda/files/minimal_trust_policy.json
new file mode 100644
index 00000000000..fb84ae9de15
--- /dev/null
+++ b/tests/integration/targets/lambda/files/minimal_trust_policy.json
@@ -0,0 +1,12 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "lambda.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
diff --git a/tests/integration/targets/lambda/tasks/main.yml b/tests/integration/targets/lambda/tasks/main.yml
index 4670a8b2544..823f479e893 100644
--- a/tests/integration/targets/lambda/tasks/main.yml
+++ b/tests/integration/targets/lambda/tasks/main.yml
@@ -8,6 +8,32 @@
   collections:
     - community.general
   block:
+  # Preparation
+  - name: create minimal lambda role
+    iam_role:
+      name: '{{ lambda_role_name }}'
+      assume_role_policy_document: '{{ lookup("file", "minimal_trust_policy.json") }}'
+      create_instance_profile: false
+      managed_policies:
+      - 'arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess'
+    register: iam_role
+  - name: wait 10 seconds for role to become available
+    pause:
+      seconds: 10
+    when: iam_role.changed
+  - name: move lambda into place for archive module
+    copy:
+      src: mini_lambda.py
+      dest: '{{ output_dir }}/mini_lambda.py'
+      mode: preserve
+  - name: bundle lambda into a zip
+    register: zip_res
+    archive:
+      format: zip
+      path: '{{ output_dir }}/mini_lambda.py'
+      dest: '{{ output_dir }}/mini_lambda.zip'
+
+  # Parameter tests
   - name: test with no parameters
     lambda:
     register: result
@@ -17,6 +43,7 @@
       that:
       - result.failed
       - 'result.msg.startswith("missing required arguments: name")'
+
   - name: test with no parameters except state absent
     lambda:
       state: absent
@@ -27,6 +54,7 @@
       that:
       - result.failed
       - 'result.msg.startswith("missing required arguments: name")'
+
   - name: test with no role or handler
     lambda:
       name: ansible-testing-fake-should-not-be-created
@@ -38,37 +66,37 @@
       that:
       - result.failed
       - 'result.msg.startswith("state is present but all of the following are missing: handler")'
-  - name: test with all module required variables but no region
+
+  - name: test state=present with security group but no vpc
     lambda:
-      name: ansible-testing-fake-should-not-be-created
-      runtime: python2.7
-      handler: no-handler
-      role: arn:fake-role-doesnt-exist
-      region: '{{ omit }}'
+      name: '{{ lambda_function_name }}'
+      runtime: 'python2.7'
+      role: '{{ lambda_role_name }}'
+      zip_file: '{{ zip_res.dest }}'
+      handler: '{{ omit }}'
+      description: '{{ omit }}'
+      vpc_subnet_ids: '{{ omit }}'
+      vpc_security_group_ids: 'sg-FA6E'
+      environment_variables: '{{ omit }}'
+      dead_letter_arn: '{{ omit }}'
     register: result
     ignore_errors: true
-  - name: assert failure when called with only 'name'
+  - name: assert lambda fails with proper message
     assert:
       that:
-      - result.failed
-      - '"requires a region and none was found" in result.msg'
-  - name: move lambda into place for archive module
-    copy:
-      src: mini_lambda.py
-      dest: '{{ output_dir }}/mini_lambda.py'
-      mode: preserve
-  - name: bundle lambda into a zip
-    register: zip_res
-    archive:
-      format: zip
-      path: '{{ output_dir }}/mini_lambda.py'
-      dest: '{{ output_dir }}/mini_lambda.zip'
+      - result is failed
+      - result.msg != "MODULE FAILURE"
+      - result.changed == False
+      - '"parameters are required together" in result.msg'
+
+  # Prepare minimal Lambda
+
   - name: test state=present - upload the lambda
     lambda:
       name: '{{ lambda_function_name }}'
       runtime: python2.7
       handler: mini_lambda.handler
-      role: ansible_lambda_role
+      role: '{{ lambda_role_name }}'
       zip_file: '{{ zip_res.dest }}'
     register: result
   - name: assert lambda upload succeeded
@@ -76,6 +104,8 @@
       that:
       - result is not failed
       - result.configuration.tracing_config.mode == "PassThrough"
+
+  # Test basic operation of Uploaded lambda
   - name: test lambda works
     execute_lambda:
       name: '{{lambda_function_name}}'
@@ -87,13 +117,15 @@
       that:
       - result is not failed
       - result.result.output.message == "hello Mr Ansible Tests"
+
+  # Test updating Lambda
   - name: test lambda config updates
     lambda:
       name: '{{lambda_function_name}}'
       runtime: nodejs10.x
       tracing_mode: Active
       handler: mini_lambda.handler
-      role: ansible_lambda_role
+      role: '{{ lambda_role_name }}'
     register: update_result
   - name: assert that update succeeded
     assert:
@@ -102,13 +134,14 @@
       - update_result.changed == True
       - update_result.configuration.runtime == 'nodejs10.x'
       - update_result.configuration.tracing_config.mode == 'Active'
-  - name: test no changes are made with the same parameters
+
+  - name: test no changes are made with the same parameters repeated
     lambda:
       name: '{{lambda_function_name}}'
       runtime: nodejs10.x
       tracing_mode: Active
       handler: mini_lambda.handler
-      role: ansible_lambda_role
+      role: '{{ lambda_role_name }}'
     register: update_result
   - name: assert that update succeeded
     assert:
@@ -117,13 +150,14 @@
       - update_result.changed == False
       - update_result.configuration.runtime == 'nodejs10.x'
       - update_result.configuration.tracing_config.mode == 'Active'
+
   - name: reset config updates for the following tests
     lambda:
       name: '{{lambda_function_name}}'
       runtime: python2.7
       tracing_mode: PassThrough
       handler: mini_lambda.handler
-      role: ansible_lambda_role
+      role: '{{ lambda_role_name }}'
     register: result
   - name: assert that reset succeeded
     assert:
@@ -132,6 +166,8 @@
       - result.changed == True
       - result.configuration.runtime == 'python2.7'
       - result.configuration.tracing_config.mode == 'PassThrough'
+
+  # Query the Lambda
   - name: lambda_info | Gather all infos for given lambda function
     lambda_info:
       name: '{{ lambda_function_name }}'
@@ -150,6 +186,7 @@
       - lambda_infos_all.function[lambda_function_name].description == ""
       - lambda_infos_all.function[lambda_function_name].function_arn is defined
       - lambda_infos_all.function[lambda_function_name].handler == "mini_lambda.handler"
+
   - name: lambda_info | Gather version infos for given lambda function
     lambda_info:
       name: '{{ lambda_function_name }}'
@@ -161,6 +198,7 @@
       - lambda_infos_versions is not failed
       - lambda_infos_versions.function[lambda_function_name].versions|length > 0
       - lambda_infos_versions.function[lambda_function_name].function_name is undefined
+
   - name: lambda_info | Gather config infos for given lambda function
     lambda_info:
       name: '{{ lambda_function_name }}'
@@ -173,6 +211,7 @@
       - lambda_infos_config.function[lambda_function_name].function_name == lambda_function_name
       - lambda_infos_config.function[lambda_function_name].description is defined
       - lambda_infos_config.function[lambda_function_name].versions is undefined
+
   - name: lambda_info | Gather policy infos for given lambda function
     lambda_info:
       name: '{{ lambda_function_name }}'
@@ -184,6 +223,7 @@
       - lambda_infos_policy is not failed
       - lambda_infos_policy.function[lambda_function_name].policy is defined
       - lambda_infos_policy.function[lambda_function_name].versions is undefined
+
   - name: lambda_info | Gather aliases infos for given lambda function
     lambda_info:
       name: '{{ lambda_function_name }}'
@@ -194,6 +234,7 @@
       that:
       - lambda_infos_aliases is not failed
       - lambda_infos_aliases.function[lambda_function_name].aliases is defined
+
   - name: lambda_info | Gather mappings infos for given lambda function
     lambda_info:
       name: '{{ lambda_function_name }}'
@@ -204,32 +245,13 @@
       that:
       - lambda_infos_mappings is not failed
       - lambda_infos_mappings.function[lambda_function_name].mappings is defined
-  - name: test state=present with security group but no vpc
-    lambda:
-      name: '{{lambda_function_name}}'
-      runtime: python2.7
-      role: ansible_lambda_role
-      zip_file: '{{zip_res.dest}}'
-      handler: '{{ omit }}'
-      description: '{{ omit }}'
-      vpc_subnet_ids: '{{ omit }}'
-      vpc_security_group_ids: sg-FA6E
-      environment_variables: '{{ omit }}'
-      dead_letter_arn: '{{ omit }}'
-    register: result
-    ignore_errors: true
-  - name: assert lambda fails with proper message
-    assert:
-      that:
-      - result is failed
-      - result.msg != "MODULE FAILURE"
-      - result.changed == False
-      - '"parameters are required together" in result.msg'
+
+  # More Lambda update tests
   - name: test state=present with all nullable variables explicitly set to null
     lambda:
       name: '{{lambda_function_name}}'
       runtime: python2.7
-      role: ansible_lambda_role
+      role: '{{ lambda_role_name }}'
       zip_file: '{{zip_res.dest}}'
       handler: mini_lambda.handler
       description: null
@@ -243,12 +265,13 @@
       that:
       - result is not failed
       - result.changed == False
+
   - name: test putting an environment variable changes lambda
     lambda:
       name: '{{lambda_function_name}}'
       runtime: python2.7
       handler: mini_lambda.handler
-      role: ansible_lambda_role
+      role: '{{ lambda_role_name }}'
       zip_file: '{{zip_res.dest}}'
       environment_variables:
         EXTRA_MESSAGE: I think you are great!!
@@ -263,47 +286,45 @@
       name: '{{lambda_function_name}}'
       payload:
         name: Mr Ansible Tests
-      security_token: '{{security_token}}'
     register: result
   - name: assert lambda manages to respond as expected
     assert:
       that:
       - result is not failed
       - result.result.output.message == "hello Mr Ansible Tests. I think you are great!!"
-  - name: test state=present triggering a network exception due to bad url
+
+  # Deletion behavious
+  - name: test state=absent (expect changed=True)
     lambda:
       name: '{{lambda_function_name}}'
-      runtime: python2.7
-      role: ansible_lambda_role
-      ec2_url: https://noexist.example.com
-      ec2_region: '{{ec2_region}}'
-      ec2_access_key: iamnotreallyanaccesskey
-      ec2_secret_key: thisisabadsecretkey
-      security_token: andthisisabadsecuritytoken
-      zip_file: '{{zip_res.dest}}'
+      state: absent
     register: result
-    ignore_errors: true
-  - name: assert lambda manages to respond as expected
+
+  - name: assert state=absent
     assert:
       that:
-      - result is failed
-      - result.changed == False
-  - name: test state=absent (expect changed=False)
+      - result is not failed
+      - result is changed
+
+  - name: test state=absent (expect changed=False) when already deleted
     lambda:
       name: '{{lambda_function_name}}'
       state: absent
     register: result
+
   - name: assert state=absent
     assert:
       that:
       - result is not failed
-      - result.changed == True
+      - result is not changed
+
+  # Parallel creations and deletions
   - name: parallel lambda creation 1/4
     lambda:
       name: '{{lambda_function_name}}_1'
       runtime: python2.7
       handler: mini_lambda.handler
-      role: ansible_lambda_role
+      role: '{{ lambda_role_name }}'
       zip_file: '{{zip_res.dest}}'
     async: 1000
     register: async_1
@@ -312,7 +333,7 @@
       name: '{{lambda_function_name}}_2'
       runtime: python2.7
       handler: mini_lambda.handler
-      role: ansible_lambda_role
+      role: '{{ lambda_role_name }}'
       zip_file: '{{zip_res.dest}}'
     async: 1000
     register: async_2
@@ -321,7 +342,7 @@
       name: '{{lambda_function_name}}_3'
       runtime: python2.7
       handler: mini_lambda.handler
-      role: ansible_lambda_role
+      role: '{{ lambda_role_name }}'
       zip_file: '{{zip_res.dest}}'
     async: 1000
     register: async_3
@@ -330,7 +351,7 @@
       name: '{{lambda_function_name}}_4'
       runtime: python2.7
       handler: mini_lambda.handler
-      role: ansible_lambda_role
+      role: '{{ lambda_role_name }}'
       zip_file: '{{zip_res.dest}}'
     register: result
   - name: assert lambda manages to respond as expected
@@ -398,9 +419,15 @@
     register: job_result
     until: job_result is finished
     retries: 30
+
   always:
   - name: ensure function is absent at end of test
     lambda:
       name: '{{lambda_function_name}}'
       state: absent
     ignore_errors: true
+  - name: ensure role has been removed at end of test
+    iam_role:
+      name: '{{ lambda_role_name }}'
+      state: absent
+    ignore_errors: true
diff --git a/tests/integration/targets/lambda_policy/defaults/main.yml b/tests/integration/targets/lambda_policy/defaults/main.yml
index db22fd7b75e..5bcb2027367 100644
--- a/tests/integration/targets/lambda_policy/defaults/main.yml
+++ b/tests/integration/targets/lambda_policy/defaults/main.yml
@@ -1,3 +1,10 @@
 ---
-# defaults file for aws_lambda test
-lambda_function_name: '{{resource_prefix}}-api-endpoint'
+# defaults file for lambda_policy integration test
+lambda_function_name: '{{ resource_prefix }}-api-endpoint'
+# IAM role names have to be less than 64 characters
+# The 8 digit identifier at the end of resource_prefix helps determine during
+# which test something was created and allows tests to be run in parallel
+# Shippable resource_prefixes are in the format shippable-123456-123, so in those cases
+# we need both sets of digits to keep the resource name unique
+unique_id: "{{ resource_prefix | regex_search('(\\d+-?)(\\d+)$') }}"
+lambda_role_name: 'ansible-test-{{ unique_id }}-lambda-policy'
diff --git a/tests/integration/targets/lambda_policy/tasks/main.yml b/tests/integration/targets/lambda_policy/tasks/main.yml
index 4714a8b2853..855e9fba994 100644
--- a/tests/integration/targets/lambda_policy/tasks/main.yml
+++ b/tests/integration/targets/lambda_policy/tasks/main.yml
@@ -1,18 +1,30 @@
-- name: Integration testing for ec2_snapshot
+- name: Integration testing for lambda_policy
+  module_defaults:
+    group/aws:
+      aws_access_key: '{{ aws_access_key }}'
+      aws_secret_key: '{{ aws_secret_key }}'
+      security_token: '{{ security_token | default(omit) }}'
+      region: '{{ aws_region }}'
   collections:
     - community.general
     - amazon.aws
 
   block:
-  - name: set up AWS credentials
-    set_fact:
-      aws_connection_info:
-        aws_region: '{{ aws_region }}'
-        aws_access_key: '{{ aws_access_key }}'
-        aws_secret_key: '{{ aws_secret_key }}'
-        security_token: '{{ security_token }}'
-    no_log: true
+  - name: create minimal lambda role
+    iam_role:
+      name: '{{ lambda_role_name }}'
+      assume_role_policy_document: '{{ lookup("file", "minimal_trust_policy.json") }}'
+      create_instance_profile: false
+      managed_policies:
+      - 'arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess'
+    register: iam_role
+  - name: wait 10 seconds for role to become available
+    pause:
+      seconds: 10
+    when: iam_role.changed
+
   - name: test with no parameters
+    module_defaults: { group/aws: {} }
     lambda_policy: null
     register: result
     ignore_errors: true
@@ -22,6 +34,7 @@
       - result.failed
       - 'result.msg.startswith("missing required arguments: ")'
   - name: test with all required dummy parameters but no region
+    module_defaults: { group/aws: {} }
     lambda_policy:
       statement_id: dummy
       principal: api_fakeway
@@ -35,6 +48,7 @@
       - result.failed
       - '"requires a region and none was found" in result.msg'
   - name: test exceptions generated by forcing bad ec2 url
+    module_defaults: { group/aws: {} }
     lambda_policy:
       function_name: '{{ lambda_function_name }}'
       state: present
@@ -68,10 +82,6 @@
       dest: '{{ output_dir }}/mini_http_lambda.zip'
   - name: create minimal lambda role
     iam_role:
-      aws_region: '{{ aws_region }}'
-      aws_access_key: '{{ aws_access_key }}'
-      aws_secret_key: '{{ aws_secret_key }}'
-      security_token: '{{ security_token }}'
       name: ansible_lambda_role
       assume_role_policy_document: '{{ lookup(''file'', ''minimal_trust_policy.json'', convert_data=False) }}'
       create_instance_profile: false
@@ -82,22 +92,14 @@
     when: iam_role.changed
   - name: test state=present - upload the lambda
     lambda:
-      aws_region: '{{ aws_region }}'
-      aws_access_key: '{{ aws_access_key }}'
-      aws_secret_key: '{{ aws_secret_key }}'
-      security_token: '{{ security_token }}'
       name: '{{lambda_function_name}}'
       runtime: python2.7
       handler: mini_http_lambda.handler
-      role: ansible_lambda_role
+      role: '{{ lambda_role_name }}'
       zip_file: '{{zip_res.dest}}'
     register: lambda_result
   - name: get the aws account ID for use in future commands
-    aws_caller_info:
-      aws_region: '{{ aws_region }}'
-      aws_access_key: '{{ aws_access_key }}'
-      aws_secret_key: '{{ aws_secret_key }}'
-      security_token: '{{ security_token }}'
+    aws_caller_info: {}
     register: aws_caller_info
   - name: register lambda uri for use in template
     set_fact:
@@ -108,10 +110,6 @@
       dest: '{{output_dir}}/endpoint-test-swagger-api.yml.j2'
   - name: deploy new API
     aws_api_gateway:
-      aws_region: '{{ aws_region }}'
-      aws_access_key: '{{ aws_access_key }}'
-      aws_secret_key: '{{ aws_secret_key }}'
-      security_token: '{{ security_token }}'
       api_file: '{{output_dir}}/endpoint-test-swagger-api.yml.j2'
       stage: lambdabased
     register: create_result
@@ -130,10 +128,6 @@
       - unauth_uri_result.status == 500
   - name: give api gateway execute permissions on lambda
     lambda_policy:
-      aws_region: '{{ aws_region }}'
-      aws_access_key: '{{ aws_access_key }}'
-      aws_secret_key: '{{ aws_secret_key }}'
-      security_token: '{{ security_token }}'
       function_name: '{{ lambda_function_name }}'
       state: present
       statement_id: api-gateway-invoke-lambdas
@@ -142,10 +136,6 @@
       source_arn: arn:aws:execute-api:{{ aws_region }}:{{ aws_caller_info.account }}:*/*
   - name: try again but with ARN
     lambda_policy:
-      aws_region: '{{ aws_region }}'
-      aws_access_key: '{{ aws_access_key }}'
-      aws_secret_key: '{{ aws_secret_key }}'
-      security_token: '{{ security_token }}'
       function_name: '{{ lambda_result.configuration.function_arn }}'
       state: present
       statement_id: api-gateway-invoke-lambdas
@@ -162,10 +152,6 @@
       - uri_result
   - name: deploy new API
     aws_api_gateway:
-      aws_region: '{{ aws_region }}'
-      aws_access_key: '{{ aws_access_key }}'
-      aws_secret_key: '{{ aws_secret_key }}'
-      security_token: '{{ security_token }}'
       api_file: '{{output_dir}}/endpoint-test-swagger-api.yml.j2'
       stage: lambdabased
     register: create_result
@@ -173,21 +159,18 @@
   always:
   - name: destroy lambda for test cleanup if created
     lambda:
-      aws_region: '{{ aws_region }}'
-      aws_access_key: '{{ aws_access_key }}'
-      aws_secret_key: '{{ aws_secret_key }}'
-      security_token: '{{ security_token }}'
       name: '{{lambda_function_name}}'
       state: absent
     register: result
     ignore_errors: true
   - name: destroy API for test cleanup if created
     aws_api_gateway:
-      aws_region: '{{ aws_region }}'
-      aws_access_key: '{{ aws_access_key }}'
-      aws_secret_key: '{{ aws_secret_key }}'
-      security_token: '{{ security_token }}'
       state: absent
       api_id: '{{api_id}}'
     register: destroy_result
     ignore_errors: true
+  - name: Clean up test role
+    iam_role:
+      name: '{{ lambda_role_name }}'
+      state: absent
+    ignore_errors: true