From d15f8e8bdff51cc42294e7d947cc86a9e81bfee9 Mon Sep 17 00:00:00 2001 From: Abhijit Menon-Sen Date: Sun, 15 Jul 2018 11:10:56 +0530 Subject: [PATCH] Retry EC2 run_instances on "Invalid IAM Instance Profile name" errors Instance creation could fail with an error like this if the IAM instance profile for the instance was created a short time before run_instances, and had not yet propagated (which happened nearly every time on fast networks): "Instance creation failed => InvalidParameterValue: Value (xxx_profile) for parameter iamInstanceProfile.name is invalid. Invalid IAM Instance Profile name" We modify ec2.py to use an "EC2Retry" wrapper that inherits from CloudRetry and wraps run_instances, detects the InvalidParameterValue exception with the 'iamInstanceProfile.name is invalid' message, and retries that request a few times. Incidentally, terraform has the same problem and used the same fix, right up to using string matching on the error message to figure out whether it's the right error or not: https://github.com/terraform-providers/terraform-provider-aws/pull/3055 --- lib/ansible/modules/cloud/amazon/ec2.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/ansible/modules/cloud/amazon/ec2.py b/lib/ansible/modules/cloud/amazon/ec2.py index e75d7e371931e0..41f8b92eae1c9e 100644 --- a/lib/ansible/modules/cloud/amazon/ec2.py +++ b/lib/ansible/modules/cloud/amazon/ec2.py @@ -576,6 +576,7 @@ from ansible.module_utils.ec2 import get_aws_connection_info, ec2_argument_spec, ec2_connect from ansible.module_utils.six import get_function_code, string_types from ansible.module_utils._text import to_bytes, to_text +from ansible.module_utils.cloud import CloudRetry try: import boto.ec2 @@ -588,6 +589,20 @@ HAS_BOTO = False +class EC2Retry(CloudRetry): + base_class = boto.exception.BotoServerError + + @staticmethod + def status_code_from_exception(e): + return [e.error_code, e.error_message] + + @staticmethod + def found(rc, catch_extra_error_codes=None): + if rc[0] == 'InvalidParameterValue' and 'iamInstanceProfile.name is invalid' in rc[1]: + return True + + return False + def find_running_instances_by_count_tag(module, ec2, vpc, count_tag, zone=None): # get reservations for instances that match tag(s) and are in the desired state @@ -959,6 +974,9 @@ def enforce_count(module, ec2, vpc): return (all_instances, instance_dict_array, changed_instance_ids, changed) +@EC2Retry.backoff(tries=3, delay=5, backoff=2.0) +def run_instances(ec2, **params): + return ec2.run_instances(**params) def create_instances(module, ec2, vpc, override_count=None): """ @@ -1165,7 +1183,7 @@ def create_instances(module, ec2, vpc, override_count=None): params['instance_initiated_shutdown_behavior'] = instance_initiated_shutdown_behavior or 'stop' try: - res = ec2.run_instances(**params) + res = run_instances(ec2, **params) except boto.exception.EC2ResponseError as e: if (params['instance_initiated_shutdown_behavior'] != 'terminate' and "InvalidParameterCombination" == e.error_code):