Merge pull request ansible-collections#616 from tremble/stability/tag…

…s/ec2_vpc_route_table

Add NotFound retries when tagging a new RouteTable

SUMMARY
Prior to botocore 1.17.24 it wasn't possible to tag the route tables as a part of the creation call. As such a separate create tags call is performed after the creation.  However, Amazon being 'Eventually consistent" we sometimes see NotFound errors just after creating a route table.
Rather than adding hard to test logic, add InvalidRouteTableID.NotFound retries.  The tests for RouteTable existing uses its own decorator, we can safely retry on InvalidRouteTableID.NotFound
Also move from custom tag handling code over to the shared code in ansible_collections.amazon.aws.plugins.module_utils.ec2
ISSUE TYPE

Bugfix Pull Request

COMPONENT NAME
ec2_vpc_route_table
ADDITIONAL INFORMATION

TASK [ec2_vpc_endpoint : Create a minimal route table (with IGW)] **************
task path: /home/zuul/.ansible/collections/ansible_collections/community/aws/tests/integration/targets/ec2_vpc_endpoint/tasks/main.yml:54
<testhost> ESTABLISH LOCAL CONNECTION FOR USER: zuul
<testhost> EXEC /bin/sh -c 'echo ~zuul && sleep 0'
<testhost> EXEC /bin/sh -c '( umask 77 && mkdir -p "` echo /home/zuul/.ansible/tmp `"&& mkdir "` echo /home/zuul/.ansible/tmp/ansible-tmp-1624964310.5067005-7213-234903855964036 `" && echo ansible-tmp-1624964310.5067005-7213-234903855964036="` echo /home/zuul/.ansible/tmp/ansible-tmp-1624964310.5067005-7213-234903855964036 `" ) && sleep 0'
Using module file /home/zuul/.ansible/collections/ansible_collections/community/aws/plugins/modules/ec2_vpc_route_table.py
<testhost> PUT /home/zuul/.ansible/tmp/ansible-local-7051vh18nelt/tmpdx500zwb TO /home/zuul/.ansible/tmp/ansible-tmp-1624964310.5067005-7213-234903855964036/AnsiballZ_ec2_vpc_route_table.py
<testhost> EXEC /bin/sh -c 'chmod u+x /home/zuul/.ansible/tmp/ansible-tmp-1624964310.5067005-7213-234903855964036/ /home/zuul/.ansible/tmp/ansible-tmp-1624964310.5067005-7213-234903855964036/AnsiballZ_ec2_vpc_route_table.py && sleep 0'
<testhost> EXEC /bin/sh -c 'ANSIBLE_DEBUG_BOTOCORE_LOGS=True /home/zuul/venv/bin/python3.6 /home/zuul/.ansible/tmp/ansible-tmp-1624964310.5067005-7213-234903855964036/AnsiballZ_ec2_vpc_route_table.py && sleep 0'
<testhost> EXEC /bin/sh -c 'rm -f -r /home/zuul/.ansible/tmp/ansible-tmp-1624964310.5067005-7213-234903855964036/ > /dev/null 2>&1 && sleep 0'
The full traceback is:
Traceback (most recent call last):
  File "/tmp/ansible_ec2_vpc_route_table_payload_y_h0b0in/ansible_ec2_vpc_route_table_payload.zip/ansible_collections/community/aws/plugins/modules/ec2_vpc_route_table.py", line 380, in ensure_tags
  File "/tmp/ansible_ec2_vpc_route_table_payload_y_h0b0in/ansible_ec2_vpc_route_table_payload.zip/ansible_collections/amazon/aws/plugins/module_utils/core.py", line 288, in deciding_wrapper
    return retrying_wrapper(*args, **kwargs)
  File "/tmp/ansible_ec2_vpc_route_table_payload_y_h0b0in/ansible_ec2_vpc_route_table_payload.zip/ansible_collections/amazon/aws/plugins/module_utils/cloud.py", line 154, in retry_func
    raise e
  File "/tmp/ansible_ec2_vpc_route_table_payload_y_h0b0in/ansible_ec2_vpc_route_table_payload.zip/ansible_collections/amazon/aws/plugins/module_utils/cloud.py", line 144, in retry_func
    return f(*args, **kwargs)
  File "/home/zuul/venv/lib/python3.6/site-packages/botocore/client.py", line 386, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/home/zuul/venv/lib/python3.6/site-packages/botocore/client.py", line 705, in _make_api_call
    raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (InvalidRouteTableID.NotFound) when calling the CreateTags operation: The routeTable ID 'rtb-08ce2a0fac35f2cc3' does not exist
fatal: [testhost]: FAILED! => {
    "boto3_version": "1.17.102",
    "botocore_version": "1.20.102",
    "changed": false,
    "error": {
        "code": "InvalidRouteTableID.NotFound",
        "message": "The routeTable ID 'rtb-08ce2a0fac35f2cc3' does not exist"
    },
    "invocation": {
        "module_args": {
            "aws_access_key": "ASIA6CCDWXDOL3X5ZIJP",
            "aws_ca_bundle": null,
            "aws_config": null,
            "aws_secret_key": "VALUE_SPECIFIED_IN_NO_LOG_PARAMETER",
            "debug_botocore_endpoint_logs": true,
            "ec2_url": null,
            "lookup": "tag",
            "profile": null,
            "propagating_vgw_ids": null,
            "purge_routes": true,
            "purge_subnets": true,
            "purge_tags": false,
            "region": "us-east-1",
            "route_table_id": null,
            "routes": [
                {
                    "destination_cidr_block": "0.0.0.0/0",
                    "gateway_id": "igw-0c6e113ca6c5d9a2c"
                }
            ],
            "security_token": "VALUE_SPECIFIED_IN_NO_LOG_PARAMETER",
            "state": "present",
            "subnets": [],
            "tags": {
                "AnsibleRun": "ansible-test-56393648-centos-8-1vcpu-vexxhost-ca-ymq-1-00016024",
                "AnsibleTest": "ec2_vpc_endpoint",
                "Name": "ansible-test-56393648-centos-8-1vcpu-vexxhost-ca-ymq-1-00016024-igw"
            },
            "validate_certs": true,
            "vpc_id": "vpc-055d3d6ce9bec0dc3"
        }
    },
    "msg": "Couldn't create tags: An error occurred (InvalidRouteTableID.NotFound) when calling the CreateTags operation: The routeTable ID 'rtb-08ce2a0fac35f2cc3' does not exist",
    "resource_actions": [
        "ec2:CreateRouteTable",
        "ec2:CreateRoute",
        "ec2:CreateTags",
        "ec2:DescribeRouteTables",
        "ec2:DescribeTags"
    ],
    "response_metadata": {
        "http_headers": {
            "cache-control": "no-cache, no-store",
            "connection": "close",
            "content-type": "text/xml;charset=UTF-8",
            "date": "Tue, 29 Jun 2021 10:58:31 GMT",
            "server": "AmazonEC2",
            "strict-transport-security": "max-age=31536000; includeSubDomains",
            "transfer-encoding": "chunked",
            "vary": "accept-encoding",
            "x-amzn-requestid": "fbefb890-4384-4001-a374-cb13db9e0d6c"
        },
        "http_status_code": 400,
        "request_id": "fbefb890-4384-4001-a374-cb13db9e0d6c",
        "retry_attempts": 0
    }
}

Reviewed-by: Jill R <None>
Reviewed-by: Alina Buzachis <None>

changelogs/fragments/616-ec2_vpc_route_table-tagging.yml

@@ -0,0 +1,4 @@
+bugfixes:
+- ec2_vpc_route_table - automatically retry when attempting to modify freshly created route tables (https://github.com/ansible-collections/community.aws/pull/616).
+minor_changes:
+- ec2_vpc_route_table - use shared code for tagging route tables (https://github.com/ansible-collections/community.aws/pull/616).

plugins/modules/ec2_vpc_route_table.py

@@ -238,10 +238,9 @@
 from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
 from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_tag_list
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import describe_ec2_tags
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ensure_ec2_tags
 from ansible_collections.amazon.aws.plugins.module_utils.waiters import get_waiter
 
 
@@ -257,14 +256,6 @@ def describe_igws_with_backoff(connection, **params):
     return paginator.paginate(**params).build_full_result()['InternetGateways']
 
 
-@AWSRetry.jittered_backoff()
-def describe_tags_with_backoff(connection, resource_id):
-    filters = ansible_dict_to_boto3_filter_list({'resource-id': resource_id})
-    paginator = connection.get_paginator('describe_tags')
-    tags = paginator.paginate(Filters=filters).build_full_result()['Tags']
-    return boto3_tag_list_to_ansible_dict(tags)
-
-
 @AWSRetry.jittered_backoff()
 def describe_route_tables_with_backoff(connection, **params):
     try:
@@ -349,45 +340,6 @@ def tags_match(match_tags, candidate_tags):
                 for k, v in match_tags.items()))
 
 
-def ensure_tags(connection=None, module=None, resource_id=None, tags=None, purge_tags=None, check_mode=None):
-    try:
-        cur_tags = describe_tags_with_backoff(connection, resource_id)
-    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-        module.fail_json_aws(e, msg='Unable to list tags for VPC')
-
-    to_add, to_delete = compare_aws_tags(cur_tags, tags, purge_tags)
-
-    if not to_add and not to_delete:
-        return {'changed': False, 'tags': cur_tags}
-    if check_mode:
-        if not purge_tags:
-            tags = cur_tags.update(tags)
-        return {'changed': True, 'tags': tags}
-
-    if to_delete:
-        try:
-            connection.delete_tags(
-                aws_retry=True,
-                Resources=[resource_id],
-                Tags=[{'Key': k} for k in to_delete])
-        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-            module.fail_json_aws(e, msg="Couldn't delete tags")
-    if to_add:
-        try:
-            connection.create_tags(
-                aws_retry=True,
-                Resources=[resource_id],
-                Tags=ansible_dict_to_boto3_tag_list(to_add))
-        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-            module.fail_json_aws(e, msg="Couldn't create tags")
-
-    try:
-        latest_tags = describe_tags_with_backoff(connection, resource_id)
-    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-        module.fail_json_aws(e, msg='Unable to list tags for VPC')
-    return {'changed': True, 'tags': latest_tags}
-
-
 def get_route_table_by_id(connection, module, route_table_id):
 
     route_table = None
@@ -410,7 +362,7 @@ def get_route_table_by_tags(connection, module, vpc_id, tags):
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg="Couldn't get route table")
     for table in route_tables:
-        this_tags = describe_tags_with_backoff(connection, table['RouteTableId'])
+        this_tags = describe_ec2_tags(connection, module, table['RouteTableId'])
         if tags_match(tags, this_tags):
             route_table = table
             count += 1
@@ -625,7 +577,7 @@ def ensure_route_table_absent(connection, module):
 def get_route_table_info(connection, module, route_table):
     result = get_route_table_by_id(connection, module, route_table['RouteTableId'])
     try:
-        result['Tags'] = describe_tags_with_backoff(connection, route_table['RouteTableId'])
+        result['Tags'] = describe_ec2_tags(connection, module, route_table['RouteTableId'])
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg="Couldn't get tags for route table")
     result = camel_dict_to_snake_dict(result, ignore_list=['Tags'])
@@ -711,10 +663,10 @@ def ensure_route_table_present(connection, module):
         changed = changed or result['changed']
 
     if not tags_valid and tags is not None:
-        result = ensure_tags(connection=connection, module=module, resource_id=route_table['RouteTableId'], tags=tags,
-                             purge_tags=purge_tags, check_mode=module.check_mode)
-        route_table['Tags'] = result['tags']
-        changed = changed or result['changed']
+        changed |= ensure_ec2_tags(connection, module, route_table['RouteTableId'],
+                                   tags=tags, purge_tags=purge_tags,
+                                   retry_codes=['InvalidRouteTableID.NotFound'])
+        route_table['Tags'] = describe_ec2_tags(connection, module, route_table['RouteTableId'])
 
     if subnets is not None:
         associated_subnets = find_subnets(connection, module, vpc_id, subnets)
@@ -751,7 +703,9 @@ def main():
                                            ['state', 'present', ['vpc_id']]],
                               supports_check_mode=True)
 
-    retry_decorator = AWSRetry.jittered_backoff(retries=10)
+    # The tests for RouteTable existing uses its own decorator, we can safely
+    # retry on InvalidRouteTableID.NotFound
+    retry_decorator = AWSRetry.jittered_backoff(retries=10, catch_extra_error_codes=['InvalidRouteTableID.NotFound'])
     connection = module.client('ec2', retry_decorator=retry_decorator)
 
     state = module.params.get('state')