From c3565bf709abf583228a2dae31f73c54ea05fdd6 Mon Sep 17 00:00:00 2001 From: Joseph Torcasso Date: Wed, 2 Mar 2022 21:34:58 -0500 Subject: [PATCH 1/5] add manual waiter for igw --- plugins/modules/ec2_vpc_igw.py | 42 ++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/plugins/modules/ec2_vpc_igw.py b/plugins/modules/ec2_vpc_igw.py index 20aaf9696ca..086938b5a45 100644 --- a/plugins/modules/ec2_vpc_igw.py +++ b/plugins/modules/ec2_vpc_igw.py @@ -110,6 +110,9 @@ from ..module_utils.ec2 import ensure_ec2_tags from ..module_utils.ec2 import ansible_dict_to_boto3_filter_list from ..module_utils.tagging import boto3_tag_list_to_ansible_dict +from time import sleep +from time import time +from random import randint class AnsibleEc2Igw(): @@ -152,6 +155,31 @@ def get_matching_igw(self, vpc_id): return igw + def wait_for_igw(self, vpc_id): + """ + Waits for existing igw to be returned via describe_internet_gateways + in get_matching_igw with exponential backoff + :param vpc_id: VPC's ID + :return igw: igw found + """ + max_backoff = 64 + timeout = 3000 + failure_counter = 0 + start_time = time() + + while True: + if time() - start_time >= timeout: + self._module.fail_json(msg='Error finding Internet Gateway in VPC {0} - please check the AWS console'.format(vpc_id)) + try: + igw = self.get_matching_igw(vpc_id) + if igw: + return igw + sleep_time = min(2 ** failure_counter + randint(1, 1000) / 1000, max_backoff) + sleep(sleep_time) + failure_counter += 1 + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + self._module.fail_json_aws(e, msg='Failure while waiting for status update') + @staticmethod def get_igw_info(igw, vpc_id): return { @@ -218,11 +246,21 @@ def ensure_igw_present(self, vpc_id, tags, purge_tags): except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: self._module.fail_json_aws(e, msg='Unable to create Internet Gateway') - self._results['changed'] |= ensure_ec2_tags( + # Ensure we can get igw object prior to modifying tags + igw = self.wait_for_igw(vpc_id) + + # Modify tags + tags_changed = ensure_ec2_tags( self._connection, self._module, igw['internet_gateway_id'], resource_type='internet-gateway', tags=tags, purge_tags=purge_tags ) - igw = self.get_matching_igw(vpc_id) + self._results['changed'] |= tags_changed + + # Wait for igw again if tags were modified to be safe + if tags_changed: + igw = self.wait_for_igw(vpc_id) + + # Update igw igw_info = self.get_igw_info(igw, vpc_id) self._results.update(igw_info) From 3842325b7269270f74160ebddb4a7ab415366745 Mon Sep 17 00:00:00 2001 From: Joseph Torcasso Date: Wed, 2 Mar 2022 21:38:29 -0500 Subject: [PATCH 2/5] changelog --- changelogs/fragments/695-ec2_vpc_igw-add-manual-waiter.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelogs/fragments/695-ec2_vpc_igw-add-manual-waiter.yml diff --git a/changelogs/fragments/695-ec2_vpc_igw-add-manual-waiter.yml b/changelogs/fragments/695-ec2_vpc_igw-add-manual-waiter.yml new file mode 100644 index 00000000000..837d0f35e39 --- /dev/null +++ b/changelogs/fragments/695-ec2_vpc_igw-add-manual-waiter.yml @@ -0,0 +1,2 @@ +bugfixes: + - ec2_vpc_igw - add manual waiter to fix 'NoneType' object is not subscriptable error (https://github.com/ansible-collections/amazon.aws/pull/695). \ No newline at end of file From 40565c3f2190acef032d825fb5d6704310459475 Mon Sep 17 00:00:00 2001 From: Alina Buzachis Date: Thu, 3 Mar 2022 21:29:47 +0100 Subject: [PATCH 3/5] Use paginator for describe internet gateways and add retry_codes='InvalidInternetGatewayID.NotFound' to ensure_ec2_tags Signed-off-by: Alina Buzachis --- plugins/modules/ec2_vpc_igw.py | 53 +++++++--------------------------- 1 file changed, 11 insertions(+), 42 deletions(-) diff --git a/plugins/modules/ec2_vpc_igw.py b/plugins/modules/ec2_vpc_igw.py index 086938b5a45..4982cf2b9eb 100644 --- a/plugins/modules/ec2_vpc_igw.py +++ b/plugins/modules/ec2_vpc_igw.py @@ -110,9 +110,12 @@ from ..module_utils.ec2 import ensure_ec2_tags from ..module_utils.ec2 import ansible_dict_to_boto3_filter_list from ..module_utils.tagging import boto3_tag_list_to_ansible_dict -from time import sleep -from time import time -from random import randint + + +@AWSRetry.jittered_backoff(retries=10, delay=10) +def describe_igws_with_backoff(connection, **params): + paginator = connection.get_paginator('describe_internet_gateways') + return paginator.paginate(**params).build_full_result()['InternetGateways'] class AnsibleEc2Igw(): @@ -138,10 +141,8 @@ def process(self): def get_matching_igw(self, vpc_id): filters = ansible_dict_to_boto3_filter_list({'attachment.vpc-id': vpc_id}) - igws = [] try: - response = self._connection.describe_internet_gateways(aws_retry=True, Filters=filters) - igws = response.get('InternetGateways', []) + igws = describe_igws_with_backoff(self._connection, Filters=filters) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: self._module.fail_json_aws(e) @@ -155,31 +156,6 @@ def get_matching_igw(self, vpc_id): return igw - def wait_for_igw(self, vpc_id): - """ - Waits for existing igw to be returned via describe_internet_gateways - in get_matching_igw with exponential backoff - :param vpc_id: VPC's ID - :return igw: igw found - """ - max_backoff = 64 - timeout = 3000 - failure_counter = 0 - start_time = time() - - while True: - if time() - start_time >= timeout: - self._module.fail_json(msg='Error finding Internet Gateway in VPC {0} - please check the AWS console'.format(vpc_id)) - try: - igw = self.get_matching_igw(vpc_id) - if igw: - return igw - sleep_time = min(2 ** failure_counter + randint(1, 1000) / 1000, max_backoff) - sleep(sleep_time) - failure_counter += 1 - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - self._module.fail_json_aws(e, msg='Failure while waiting for status update') - @staticmethod def get_igw_info(igw, vpc_id): return { @@ -239,28 +215,21 @@ def ensure_igw_present(self, vpc_id, tags, purge_tags): # Ensure the gateway is attached before proceeding waiter = get_waiter(self._connection, 'internet_gateway_attached') waiter.wait(InternetGatewayIds=[igw['internet_gateway_id']]) - self._results['changed'] = True except botocore.exceptions.WaiterError as e: self._module.fail_json_aws(e, msg="No Internet Gateway exists.") except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: self._module.fail_json_aws(e, msg='Unable to create Internet Gateway') - # Ensure we can get igw object prior to modifying tags - igw = self.wait_for_igw(vpc_id) - # Modify tags - tags_changed = ensure_ec2_tags( + self._results['changed'] |= ensure_ec2_tags( self._connection, self._module, igw['internet_gateway_id'], - resource_type='internet-gateway', tags=tags, purge_tags=purge_tags + resource_type='internet-gateway', tags=tags, purge_tags=purge_tags, + retry_codes='InvalidInternetGatewayID.NotFound' ) - self._results['changed'] |= tags_changed - - # Wait for igw again if tags were modified to be safe - if tags_changed: - igw = self.wait_for_igw(vpc_id) # Update igw + igw = self.get_matching_igw(vpc_id) igw_info = self.get_igw_info(igw, vpc_id) self._results.update(igw_info) From 0fcd040de542da4d6190e9f3e26472a56c250fb9 Mon Sep 17 00:00:00 2001 From: Joseph Torcasso Date: Thu, 3 Mar 2022 18:31:47 -0500 Subject: [PATCH 4/5] update changelog --- changelogs/fragments/695-ec2_vpc_igw-add-manual-waiter.yml | 2 -- .../fragments/695-ec2_vpc_igw-fix-nonetype-with-paginator.yml | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 changelogs/fragments/695-ec2_vpc_igw-add-manual-waiter.yml create mode 100644 changelogs/fragments/695-ec2_vpc_igw-fix-nonetype-with-paginator.yml diff --git a/changelogs/fragments/695-ec2_vpc_igw-add-manual-waiter.yml b/changelogs/fragments/695-ec2_vpc_igw-add-manual-waiter.yml deleted file mode 100644 index 837d0f35e39..00000000000 --- a/changelogs/fragments/695-ec2_vpc_igw-add-manual-waiter.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - ec2_vpc_igw - add manual waiter to fix 'NoneType' object is not subscriptable error (https://github.com/ansible-collections/amazon.aws/pull/695). \ No newline at end of file diff --git a/changelogs/fragments/695-ec2_vpc_igw-fix-nonetype-with-paginator.yml b/changelogs/fragments/695-ec2_vpc_igw-fix-nonetype-with-paginator.yml new file mode 100644 index 00000000000..eb0d9b1fe15 --- /dev/null +++ b/changelogs/fragments/695-ec2_vpc_igw-fix-nonetype-with-paginator.yml @@ -0,0 +1,2 @@ +bugfixes: + - ec2_vpc_igw - use paginator for describe internet gateways and add retry to fix NoneType' object is not subscriptable error (https://github.com/ansible-collections/amazon.aws/pull/695). From a0853f998974b8c19ebecb86f955c16046f38402 Mon Sep 17 00:00:00 2001 From: Joseph Torcasso Date: Thu, 3 Mar 2022 18:34:15 -0500 Subject: [PATCH 5/5] fix typo in changelog --- .../fragments/695-ec2_vpc_igw-fix-nonetype-with-paginator.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelogs/fragments/695-ec2_vpc_igw-fix-nonetype-with-paginator.yml b/changelogs/fragments/695-ec2_vpc_igw-fix-nonetype-with-paginator.yml index eb0d9b1fe15..ff20ab02825 100644 --- a/changelogs/fragments/695-ec2_vpc_igw-fix-nonetype-with-paginator.yml +++ b/changelogs/fragments/695-ec2_vpc_igw-fix-nonetype-with-paginator.yml @@ -1,2 +1,2 @@ bugfixes: - - ec2_vpc_igw - use paginator for describe internet gateways and add retry to fix NoneType' object is not subscriptable error (https://github.com/ansible-collections/amazon.aws/pull/695). + - ec2_vpc_igw - use paginator for describe internet gateways and add retry to fix NoneType object is not subscriptable error (https://github.com/ansible-collections/amazon.aws/pull/695).