Yet more integration test cleanup and migration to module_defaults (a…

…nsible-collections#96)

* Mass-migrate integration tests over to module_defaults instead of set_fact hack

* More test cleanup

* Rename templates to .yml.j2

* Remove the ec2_url tests - they weren't actually testing the use of ec2_url

- Because we're also connecting to the VPC and ELB endpoints region was mandatory
- When region is passed it takes precedence over ec2_url
- The error that was actually being generated was caused by missing tokens, not by a bad URL.

* Try to reduce the VPC test errors

plugins/lookup/aws_account_attribute.py

@@ -92,7 +92,8 @@ def _get_credentials(options):
     credentials['aws_profile'] = options['aws_profile']
     credentials['aws_secret_access_key'] = options['aws_secret_key']
     credentials['aws_access_key_id'] = options['aws_access_key']
-    credentials['aws_session_token'] = options['aws_security_token']
+    if options['aws_security_token']:
+        credentials['aws_session_token'] = options['aws_security_token']
 
     return credentials
 

plugins/modules/ec2_vpc_net.py

@@ -271,7 +271,8 @@ def update_vpc_tags(connection, module, vpc_id, tags, name):
     tags.update({'Name': name})
     tags = dict((k, to_native(v)) for k, v in tags.items())
     try:
-        current_tags = dict((t['Key'], t['Value']) for t in connection.describe_tags(Filters=[{'Name': 'resource-id', 'Values': [vpc_id]}])['Tags'])
+        filters = [{'Name': 'resource-id', 'Values': [vpc_id]}]
+        current_tags = dict((t['Key'], t['Value']) for t in connection.describe_tags(Filters=filters, aws_retry=True)['Tags'])
         tags_to_update, dummy = compare_aws_tags(current_tags, tags, False)
         if tags_to_update:
             if not module.check_mode:
@@ -294,7 +295,7 @@ def update_dhcp_opts(connection, module, vpc_obj, dhcp_id):
     if vpc_obj['DhcpOptionsId'] != dhcp_id:
         if not module.check_mode:
             try:
-                connection.associate_dhcp_options(DhcpOptionsId=dhcp_id, VpcId=vpc_obj['VpcId'])
+                connection.associate_dhcp_options(DhcpOptionsId=dhcp_id, VpcId=vpc_obj['VpcId'], aws_retry=True)
             except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 module.fail_json_aws(e, msg="Failed to associate DhcpOptionsId {0}".format(dhcp_id))
 
@@ -313,7 +314,7 @@ def update_dhcp_opts(connection, module, vpc_obj, dhcp_id):
 def create_vpc(connection, module, cidr_block, tenancy):
     try:
         if not module.check_mode:
-            vpc_obj = connection.create_vpc(CidrBlock=cidr_block, InstanceTenancy=tenancy)
+            vpc_obj = connection.create_vpc(CidrBlock=cidr_block, InstanceTenancy=tenancy, aws_retry=True)
         else:
             module.exit_json(changed=True)
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
@@ -334,7 +335,8 @@ def wait_for_vpc_attribute(connection, module, vpc_id, attribute, expected_value
     while time() < start_time + 300:
         current_value = connection.describe_vpc_attribute(
             Attribute=attribute,
-            VpcId=vpc_id
+            VpcId=vpc_id,
+            aws_retry=True
         )['{0}{1}'.format(attribute[0].upper(), attribute[1:])]['Value']
         if current_value != expected_value:
             sleep(3)
@@ -428,7 +430,7 @@ def main():
             for cidr in to_add:
                 changed = True
                 try:
-                    connection.associate_vpc_cidr_block(CidrBlock=cidr, VpcId=vpc_id)
+                    connection.associate_vpc_cidr_block(CidrBlock=cidr, VpcId=vpc_id, aws_retry=True)
                 except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                     module.fail_json_aws(e, "Unable to associate CIDR {0}.".format(ipv6_cidr))
         if ipv6_cidr:
@@ -438,7 +440,7 @@ def main():
                     vpc_obj['Ipv6CidrBlockAssociationSet'][0]['Ipv6CidrBlock']))
             else:
                 try:
-                    connection.associate_vpc_cidr_block(AmazonProvidedIpv6CidrBlock=ipv6_cidr, VpcId=vpc_id)
+                    connection.associate_vpc_cidr_block(AmazonProvidedIpv6CidrBlock=ipv6_cidr, VpcId=vpc_id, aws_retry=True)
                     changed = True
                 except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                     module.fail_json_aws(e, "Unable to associate CIDR {0}.".format(ipv6_cidr))
@@ -447,7 +449,7 @@ def main():
             for association_id in to_remove:
                 changed = True
                 try:
-                    connection.disassociate_vpc_cidr_block(AssociationId=association_id)
+                    connection.disassociate_vpc_cidr_block(AssociationId=association_id, aws_retry=True)
                 except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                     module.fail_json_aws(e, "Unable to disassociate {0}. You must detach or delete all gateways and resources that "
                                          "are associated with the CIDR block before you can disassociate it.".format(association_id))
@@ -472,14 +474,14 @@ def main():
             changed = True
             if not module.check_mode:
                 try:
-                    connection.modify_vpc_attribute(VpcId=vpc_id, EnableDnsSupport={'Value': dns_support})
+                    connection.modify_vpc_attribute(VpcId=vpc_id, EnableDnsSupport={'Value': dns_support}, aws_retry=True)
                 except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                     module.fail_json_aws(e, "Failed to update enabled dns support attribute")
         if current_dns_hostnames != dns_hostnames:
             changed = True
             if not module.check_mode:
                 try:
-                    connection.modify_vpc_attribute(VpcId=vpc_id, EnableDnsHostnames={'Value': dns_hostnames})
+                    connection.modify_vpc_attribute(VpcId=vpc_id, EnableDnsHostnames={'Value': dns_hostnames}, aws_retry=True)
                 except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                     module.fail_json_aws(e, "Failed to update enabled dns hostnames attribute")
 
@@ -511,7 +513,7 @@ def main():
         if vpc_id is not None:
             try:
                 if not module.check_mode:
-                    connection.delete_vpc(VpcId=vpc_id)
+                    connection.delete_vpc(VpcId=vpc_id, aws_retry=True)
                 changed = True
             except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 module.fail_json_aws(e, msg="Failed to delete VPC {0} You may want to use the ec2_vpc_subnet, ec2_vpc_igw, "

plugins/modules/ec2_vpc_net_info.py

@@ -164,19 +164,6 @@
 from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict
 
 
-@AWSRetry.exponential_backoff()
-def describe_vpc_attr_with_backoff(connection, vpc_id, vpc_attribute):
-    """
-    Describe VPC Attributes with AWSRetry backoff throttling support.
-
-    connection  : boto3 client connection object
-    vpc_id      : The VPC ID to pull attribute value from
-    vpc_attribute     : The VPC attribute to get the value from - valid options = enableDnsSupport or enableDnsHostnames
-    """
-
-    return connection.describe_vpc_attribute(VpcId=vpc_id, Attribute=vpc_attribute)
-
-
 def describe_vpcs(connection, module):
     """
     Describe VPCs.
@@ -204,14 +191,14 @@ def describe_vpcs(connection, module):
 
     # We can get these results in bulk but still needs two separate calls to the API
     try:
-        cl_enabled = connection.describe_vpc_classic_link(VpcIds=vpc_list)
+        cl_enabled = connection.describe_vpc_classic_link(VpcIds=vpc_list, aws_retry=True)
     except is_boto3_error_code('UnsupportedOperation'):
         cl_enabled = {'Vpcs': [{'VpcId': vpc_id, 'ClassicLinkEnabled': False} for vpc_id in vpc_list]}
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e, msg='Unable to describe if ClassicLink is enabled')
 
     try:
-        cl_dns_support = connection.describe_vpc_classic_link_dns_support(VpcIds=vpc_list)
+        cl_dns_support = connection.describe_vpc_classic_link_dns_support(VpcIds=vpc_list, aws_retry=True)
     except is_boto3_error_code('UnsupportedOperation'):
         cl_dns_support = {'Vpcs': [{'VpcId': vpc_id, 'ClassicLinkDnsSupported': False} for vpc_id in vpc_list]}
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
@@ -222,11 +209,13 @@ def describe_vpcs(connection, module):
         error_message = "Unable to describe VPC attribute {0}"
         # We have to make two separate calls per VPC to get these attributes.
         try:
-            dns_support = describe_vpc_attr_with_backoff(connection, vpc['VpcId'], 'enableDnsSupport')
+            dns_support = connection.describe_vpc_attribute(VpcId=vpc['VpcId'],
+                                                            Attribute='enableDnsSupport', aws_retry=True)
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e, msg=error_message.format('enableDnsSupport'))
         try:
-            dns_hostnames = describe_vpc_attr_with_backoff(connection, vpc['VpcId'], 'enableDnsHostnames')
+            dns_hostnames = connection.describe_vpc_attribute(VpcId=vpc['VpcId'],
+                                                              Attribute='enableDnsHostnames', aws_retry=True)
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e, msg=error_message.format('enableDnsHostnames'))
 
@@ -262,7 +251,7 @@ def main():
     if module._name == 'ec2_vpc_net_facts':
         module.deprecate("The 'ec2_vpc_net_facts' module has been renamed to 'ec2_vpc_net_info'", date='2021-12-01', collection_name='amazon.aws')
 
-    connection = module.client('ec2')
+    connection = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff(retries=10))
 
     describe_vpcs(connection, module)
 

tests/integration/targets/aws_s3/tasks/main.yml

@@ -1,5 +1,5 @@
 ---
-# Test suite for aws_s3
+# Integration tests for aws_s3
 - module_defaults:
     group/aws:
       aws_access_key: "{{ aws_access_key }}"