diff --git a/plugins/module_utils/cloud.py b/plugins/module_utils/cloud.py index fdb7fdb1b58..600a7ed158b 100644 --- a/plugins/module_utils/cloud.py +++ b/plugins/module_utils/cloud.py @@ -1,221 +1,206 @@ +# Copyright (c) 2021 Ansible Project # -# (c) 2016 Allen Sanabria, +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. # -# This file is part of Ansible +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: # -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. # -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # + from __future__ import (absolute_import, division, print_function) __metaclass__ = type -""" -This module adds shared support for generic cloud modules - -In order to use this module, include it as part of a custom -module as shown below. - -from ansible.module_utils.cloud import CloudRetry - -The 'cloud' module provides the following common classes: - - * CloudRetry - - The base class to be used by other cloud providers, in order to - provide a backoff/retry decorator based on status codes. - - - Example using the AWSRetry class which inherits from CloudRetry. - - @AWSRetry.exponential_backoff(retries=10, delay=3) - get_ec2_security_group_ids_from_names() - - @AWSRetry.jittered_backoff() - get_ec2_security_group_ids_from_names() - -""" -import random -from functools import wraps -import syslog import time +import functools +import random -def _exponential_backoff(retries=10, delay=2, backoff=2, max_delay=60): - """ Customizable exponential backoff strategy. +class BackoffIterator: + """iterate sleep value based on the exponential or jitter back-off algorithm. Args: - retries (int): Maximum number of times to retry a request. - delay (float): Initial (base) delay. - backoff (float): base of the exponent to use for exponential - backoff. - max_delay (int): Optional. If provided each delay generated is capped - at this amount. Defaults to 60 seconds. - Returns: - Callable that returns a generator. This generator yields durations in - seconds to be used as delays for an exponential backoff strategy. - Usage: - >>> backoff = _exponential_backoff() - >>> backoff - - >>> list(backoff()) - [2, 4, 8, 16, 32, 60, 60, 60, 60, 60] + delay (int or float): initial delay. + backoff (int or float): backoff multiplier e.g. value of 2 will double the delay each retry. + max_delay (int or None): maximum amount of time to wait between retries. + jitter (bool): if set to true, add jitter to the generate value. """ - def backoff_gen(): - for retry in range(0, retries): - sleep = delay * backoff ** retry - yield sleep if max_delay is None else min(sleep, max_delay) - return backoff_gen - -def _full_jitter_backoff(retries=10, delay=3, max_delay=60, _random=random): - """ Implements the "Full Jitter" backoff strategy described here - https://www.awsarchitectureblog.com/2015/03/backoff.html - Args: - retries (int): Maximum number of times to retry a request. - delay (float): Approximate number of seconds to sleep for the first - retry. - max_delay (int): The maximum number of seconds to sleep for any retry. - _random (random.Random or None): Makes this generator testable by - allowing developers to explicitly pass in the a seeded Random. - Returns: - Callable that returns a generator. This generator yields durations in - seconds to be used as delays for a full jitter backoff strategy. - Usage: - >>> backoff = _full_jitter_backoff(retries=5) - >>> backoff - - >>> list(backoff()) - [3, 6, 5, 23, 38] - >>> list(backoff()) - [2, 1, 6, 6, 31] + def __init__(self, delay, backoff, max_delay=None, jitter=False): + self.delay = delay + self.backoff = backoff + self.max_delay = max_delay + self.jitter = jitter + + def __iter__(self): + self.current_delay = self.delay + return self + + def __next__(self): + return_value = self.current_delay if self.max_delay is None else min(self.current_delay, self.max_delay) + if self.jitter: + return_value = random.uniform(0.0, return_value) + self.current_delay *= self.backoff + return return_value + + +def _retry_func(func, sleep_time_generator, retries, catch_extra_error_codes, found_f, status_code_from_except_f, base_class): + counter = 0 + for sleep_time in sleep_time_generator: + try: + return func() + except Exception as exc: + counter += 1 + if counter == retries: + raise + if base_class and not isinstance(exc, base_class): + raise + status_code = status_code_from_except_f(exc) + if found_f(status_code, catch_extra_error_codes): + time.sleep(sleep_time) + else: + raise + + +class CloudRetry: """ - def backoff_gen(): - for retry in range(0, retries): - yield _random.randint(0, min(max_delay, delay * 2 ** retry)) - return backoff_gen - - -class CloudRetry(object): - """ CloudRetry can be used by any cloud provider, in order to implement a - backoff algorithm/retry effect based on Status Code from Exceptions. + The base class to be used by other cloud providers to provide a backoff/retry decorator based on status codes. """ - # This is the base class of the exception. - # AWS Example botocore.exceptions.ClientError - # NoneType can't be raised (it's not a subclass of Exception) so would never be caught by an except. + base_class = type(None) @staticmethod def status_code_from_exception(error): - """ Return the status code from the exception object + """ + Returns the Error 'code' from an exception. Args: - error (object): The exception itself. + error: The Exception from which the error code is to be extracted. + error will be an instance of class.base_class. """ - pass + raise NotImplementedError() @staticmethod def found(response_code, catch_extra_error_codes=None): - """ Return True if the Response Code to retry on was found. - Args: - response_code (str): This is the Response Code that is being matched against. - """ - pass + def _is_iterable(): + try: + it = iter(catch_extra_error_codes) + except TypeError: + # not iterable + return False + else: + # iterable + return True + return _is_iterable() and response_code in catch_extra_error_codes @classmethod - def _backoff(cls, backoff_strategy, catch_extra_error_codes=None): - """ Retry calling the Cloud decorated function using the provided - backoff strategy. - Args: - backoff_strategy (callable): Callable that returns a generator. The - generator should yield sleep times for each retry of the decorated - function. - """ - def deco(f): - @wraps(f) - def retry_func(*args, **kwargs): - for delay in backoff_strategy(): - try: - return f(*args, **kwargs) - except Exception as e: - if isinstance(e, cls.base_class): - response_code = cls.status_code_from_exception(e) - if cls.found(response_code, catch_extra_error_codes): - msg = "{0}: Retrying in {1} seconds...".format(str(e), delay) - syslog.syslog(syslog.LOG_INFO, msg) - time.sleep(delay) - else: - # Return original exception if exception is not a ClientError - raise e - else: - # Return original exception if exception is not a ClientError - raise e - return f(*args, **kwargs) - - return retry_func # true decorator - - return deco + def base_decorator(cls, retries, found, status_code_from_exception, catch_extra_error_codes, sleep_time_generator): + def retry_decorator(func): + @functools.wraps(func) + def _retry_wrapper(*args, **kwargs): + partial_func = functools.partial(func, *args, **kwargs) + return _retry_func( + func=partial_func, + sleep_time_generator=sleep_time_generator, + retries=retries, + catch_extra_error_codes=catch_extra_error_codes, + found_f=found, + status_code_from_except_f=status_code_from_exception, + base_class=cls.base_class, + ) + return _retry_wrapper + return retry_decorator @classmethod def exponential_backoff(cls, retries=10, delay=3, backoff=2, max_delay=60, catch_extra_error_codes=None): - """ - Retry calling the Cloud decorated function using an exponential backoff. - - Kwargs: + """Wrap a callable with retry behavior. + Args: retries (int): Number of times to retry a failed request before giving up default=10 delay (int or float): Initial delay between retries in seconds default=3 - backoff (int or float): backoff multiplier e.g. value of 2 will - double the delay each retry - default=1.1 + backoff (int or float): backoff multiplier e.g. value of 2 will double the delay each retry + default=2 max_delay (int or None): maximum amount of time to wait between retries. default=60 + catch_extra_error_codes: Additional error messages to catch, in addition to those which may be defined by a subclass of CloudRetry + default=None + Returns: + Callable: A generator that calls the decorated function using an exponential backoff. """ - return cls._backoff(_exponential_backoff( - retries=retries, delay=delay, backoff=backoff, max_delay=max_delay), catch_extra_error_codes) + sleep_time_generator = BackoffIterator(delay=delay, backoff=backoff, max_delay=max_delay) + return cls.base_decorator( + retries=retries, + found=cls.found, + status_code_from_exception=cls.status_code_from_exception, + catch_extra_error_codes=catch_extra_error_codes, + sleep_time_generator=sleep_time_generator, + ) @classmethod - def jittered_backoff(cls, retries=10, delay=3, max_delay=60, catch_extra_error_codes=None): - """ - Retry calling the Cloud decorated function using a jittered backoff - strategy. More on this strategy here: - - https://www.awsarchitectureblog.com/2015/03/backoff.html - - Kwargs: + def jittered_backoff(cls, retries=10, delay=3, backoff=2.0, max_delay=60, catch_extra_error_codes=None): + """Wrap a callable with retry behavior. + Args: retries (int): Number of times to retry a failed request before giving up default=10 - delay (int): Initial delay between retries in seconds + delay (int or float): Initial delay between retries in seconds default=3 - max_delay (int): maximum amount of time to wait between retries. + backoff (int or float): backoff multiplier e.g. value of 2 will double the delay each retry + default=2.0 + max_delay (int or None): maximum amount of time to wait between retries. default=60 + catch_extra_error_codes: Additional error messages to catch, in addition to those which may be defined by a subclass of CloudRetry + default=None + Returns: + Callable: A generator that calls the decorated function using using a jittered backoff strategy. """ - return cls._backoff(_full_jitter_backoff( - retries=retries, delay=delay, max_delay=max_delay), catch_extra_error_codes) + sleep_time_generator = BackoffIterator(delay=delay, backoff=backoff, max_delay=max_delay, jitter=True) + return cls.base_decorator( + retries=retries, + found=cls.found, + status_code_from_exception=cls.status_code_from_exception, + catch_extra_error_codes=catch_extra_error_codes, + sleep_time_generator=sleep_time_generator, + ) @classmethod def backoff(cls, tries=10, delay=3, backoff=1.1, catch_extra_error_codes=None): """ - Retry calling the Cloud decorated function using an exponential backoff. - - Compatibility for the original implementation of CloudRetry.backoff that - did not provide configurable backoff strategies. Developers should use - CloudRetry.exponential_backoff instead. - - Kwargs: - tries (int): Number of times to try (not retry) before giving up + Wrap a callable with retry behavior. + Developers should use CloudRetry.exponential_backoff instead. + This method has been deprecated and will be removed in release 4.0.0, consider using exponential_backoff method instead. + Args: + retries (int): Number of times to retry a failed request before giving up default=10 delay (int or float): Initial delay between retries in seconds default=3 - backoff (int or float): backoff multiplier e.g. value of 2 will - double the delay each retry + backoff (int or float): backoff multiplier e.g. value of 2 will double the delay each retry default=1.1 + catch_extra_error_codes: Additional error messages to catch, in addition to those which may be defined by a subclass of CloudRetry + default=None + Returns: + Callable: A generator that calls the decorated function using an exponential backoff. """ return cls.exponential_backoff( - retries=tries - 1, delay=delay, backoff=backoff, max_delay=None, catch_extra_error_codes=catch_extra_error_codes) + retries=tries, + delay=delay, + backoff=backoff, + max_delay=None, + catch_extra_error_codes=catch_extra_error_codes, + ) diff --git a/tests/unit/module_utils/test_cloud.py b/tests/unit/module_utils/test_cloud.py new file mode 100644 index 00000000000..f54dc2dc66e --- /dev/null +++ b/tests/unit/module_utils/test_cloud.py @@ -0,0 +1,263 @@ +# (c) 2021 Red Hat Inc. +# +# This file is part of Ansible +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible_collections.amazon.aws.plugins.module_utils.cloud import CloudRetry, BackoffIterator +import unittest +import random +from datetime import datetime + + +def test_backoff_value_generator(): + max_delay = 60 + initial = 3 + backoff = 2 + + min_sleep = initial + counter = 0 + for sleep in BackoffIterator(delay=initial, backoff=backoff, max_delay=max_delay): + if counter > 4: + assert sleep == max_delay + else: + assert sleep == min_sleep + min_sleep *= backoff + counter += 1 + if counter == 10: + break + + +def test_backoff_value_generator_with_jitter(): + max_delay = 60 + initial = 3 + backoff = 2 + + min_sleep = initial + counter = 0 + for sleep in BackoffIterator(delay=initial, backoff=backoff, max_delay=max_delay, jitter=True): + if counter > 4: + assert sleep <= max_delay + else: + assert sleep <= min_sleep + min_sleep *= backoff + counter += 1 + if counter == 10: + break + + +class CloudRetryUtils(unittest.TestCase): + + error_codes = [400, 500, 600] + custom_error_codes = [100, 200, 300] + + class TestException(Exception): + """ + custom exception class for testing + """ + def __init__(self, status): + self.status = status + + def __str__(self): + return "TestException with status: {0}".format(self.status) + + class UnitTestsRetry(CloudRetry): + base_class = Exception + + @staticmethod + def status_code_from_exception(error): + return getattr(error, "status") if hasattr(error, "status") else None + + class CustomRetry(CloudRetry): + base_class = Exception + + @staticmethod + def status_code_from_exception(error): + return error.status['response']['status'] + + @staticmethod + def found(response_code, catch_extra_error_codes=None): + if catch_extra_error_codes: + return response_code in catch_extra_error_codes + CloudRetryUtils.custom_error_codes + else: + return response_code in CloudRetryUtils.custom_error_codes + + class KeyRetry(CloudRetry): + base_class = KeyError + + @staticmethod + def status_code_from_exception(error): + return True + + @staticmethod + def found(response_code, catch_extra_error_codes=None): + return True + + class KeyAndIndexRetry(CloudRetry): + base_class = (KeyError, IndexError) + + @staticmethod + def status_code_from_exception(error): + return True + + @staticmethod + def found(response_code, catch_extra_error_codes=None): + return True + + # ======================================================== + # Setup some initial data that we can use within our tests + # ======================================================== + def setUp(self): + # nothing to do on setup stage + pass + + # ======================================================== + # retry exponential backoff + # ======================================================== + def test_retry_exponential_backoff(self): + + @CloudRetryUtils.UnitTestsRetry.exponential_backoff(retries=3, delay=1, backoff=1.1, max_delay=3, catch_extra_error_codes=CloudRetryUtils.error_codes) + def test_retry_func(): + if test_retry_func.counter < 2: + test_retry_func.counter += 1 + raise self.TestException(status=random.choice(CloudRetryUtils.error_codes)) + else: + return True + + test_retry_func.counter = 0 + ret = test_retry_func() + assert ret is True + + def test_retry_exponential_backoff_with_unexpected_exception(self): + unexpected_except = self.TestException(status=100) + + @CloudRetryUtils.UnitTestsRetry.exponential_backoff(retries=3, delay=1, backoff=1.1, max_delay=3, catch_extra_error_codes=CloudRetryUtils.error_codes) + def test_retry_func(): + if test_retry_func.counter == 0: + test_retry_func.counter += 1 + raise self.TestException(status=random.choice(CloudRetryUtils.error_codes)) + else: + raise unexpected_except + + test_retry_func.counter = 0 + try: + ret = test_retry_func() + except self.TestException as exc: + assert exc.status == unexpected_except.status + + # ======================================================== + # retry jittered backoff + # ======================================================== + def test_retry_jitter_backoff(self): + @CloudRetryUtils.UnitTestsRetry.jittered_backoff(retries=3, delay=1, max_delay=3, catch_extra_error_codes=CloudRetryUtils.error_codes) + def test_retry_func(): + if test_retry_func.counter < 2: + test_retry_func.counter += 1 + raise self.TestException(status=random.choice(CloudRetryUtils.error_codes)) + else: + return True + + test_retry_func.counter = 0 + ret = test_retry_func() + assert ret is True + + def test_retry_jittered_backoff_with_unexpected_exception(self): + unexpected_except = self.TestException(status=100) + + @CloudRetryUtils.UnitTestsRetry.jittered_backoff(retries=3, delay=1, max_delay=3, catch_extra_error_codes=CloudRetryUtils.error_codes) + def test_retry_func(): + if test_retry_func.counter == 0: + test_retry_func.counter += 1 + raise self.TestException(status=random.choice(CloudRetryUtils.error_codes)) + else: + raise unexpected_except + + test_retry_func.counter = 0 + try: + ret = test_retry_func() + except self.TestException as exc: + assert exc.status == unexpected_except.status + + # ======================================================== + # retry with custom class + # ======================================================== + def test_retry_exponential_backoff_custom_class(self): + def build_response(): + return dict(response=dict(status=random.choice(CloudRetryUtils.custom_error_codes))) + + @self.CustomRetry.exponential_backoff(retries=3, delay=1, backoff=1.1, max_delay=3, catch_extra_error_codes=CloudRetryUtils.error_codes) + def test_retry_func(): + if test_retry_func.counter < 2: + test_retry_func.counter += 1 + raise self.TestException(build_response()) + else: + return True + + test_retry_func.counter = 0 + + ret = test_retry_func() + assert ret is True + + # ============================================================= + # Test wrapped function multiple times will restart the sleep + # ============================================================= + def test_wrapped_function_called_several_times(self): + @CloudRetryUtils.UnitTestsRetry.exponential_backoff(retries=2, delay=2, backoff=4, max_delay=100, catch_extra_error_codes=CloudRetryUtils.error_codes) + def _fail(): + raise self.TestException(status=random.choice(CloudRetryUtils.error_codes)) + + # run the method 3 times and assert that each it is retrying after 2secs + # the elapsed execution time should be closed to 2sec + for u in range(3): + start = datetime.now() + raised = False + try: + _fail() + except self.TestException: + raised = True + duration = (datetime.now() - start).seconds + assert duration == 2 + finally: + assert raised + + def test_only_base_exception(self): + def _fail_index(): + my_list = list() + return my_list[5] + + def _fail_key(): + my_dict = dict() + return my_dict['invalid_key'] + + def _fail_exception(): + raise Exception('bang') + + key_retry_decorator = CloudRetryUtils.KeyRetry.exponential_backoff(retries=2, delay=2, backoff=4, max_delay=100) + key_and_index_retry_decorator = CloudRetryUtils.KeyAndIndexRetry.exponential_backoff(retries=2, delay=2, backoff=4, max_delay=100) + + expectations = [ + [key_retry_decorator, _fail_exception, 0], + [key_retry_decorator, _fail_index, 0], + [key_retry_decorator, _fail_key, 2], + [key_and_index_retry_decorator, _fail_exception, 0], + [key_and_index_retry_decorator, _fail_index, 2], + [key_and_index_retry_decorator, _fail_key, 2], + ] + + for expection in expectations: + decorator = expection[0] + function = expection[1] + duration = expection[2] + + start = datetime.now() + raised = False + try: + decorator(function)() + except Exception: + raised = True + _duration = (datetime.now() - start).seconds + assert duration == _duration + finally: + assert raised