From 3efd3de57656cf6ecd17a135916f4b2c1cb5edc1 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Mon, 31 Oct 2022 10:25:41 +0100 Subject: [PATCH] Move RetryingBotoClientWrapper into module_utils.retries This means we can use it later with non-module plugins --- ...0221101-move-RetryingBotoClientWrapper.yml | 2 + plugins/module_utils/exceptions.py | 38 +++++++++++++++++++ plugins/module_utils/modules.py | 38 +------------------ plugins/module_utils/retries.py | 35 +++++++++++++++++ plugins/module_utils/waiters.py | 4 +- .../plugins/modules/test_cloudformation.py | 4 +- 6 files changed, 81 insertions(+), 40 deletions(-) create mode 100644 changelogs/fragments/20221101-move-RetryingBotoClientWrapper.yml create mode 100644 plugins/module_utils/exceptions.py diff --git a/changelogs/fragments/20221101-move-RetryingBotoClientWrapper.yml b/changelogs/fragments/20221101-move-RetryingBotoClientWrapper.yml new file mode 100644 index 00000000000..cec351eaca4 --- /dev/null +++ b/changelogs/fragments/20221101-move-RetryingBotoClientWrapper.yml @@ -0,0 +1,2 @@ +minor_changes: +- module_utils - moves RetryingBotoClientWrapper into module.retries so it's available for other plugin types (). diff --git a/plugins/module_utils/exceptions.py b/plugins/module_utils/exceptions.py new file mode 100644 index 00000000000..a32afc3b52c --- /dev/null +++ b/plugins/module_utils/exceptions.py @@ -0,0 +1,38 @@ +# (c) 2022 Red Hat Inc. +# +# This file is part of Ansible +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.module_utils._text import to_native + + +class AnsibleAWSError(Exception): + + def __str__(self): + if self.exception and self.message: + return "{0}: {1}".format(self.message, to_native(self.exception)) + + return super(AnsibleAWSError, self).__str__() + + def __init__(self, message=None, exception=None, **kwargs): + if not message and not exception: + super(AnsibleAWSError, self).__init__() + if not message: + super(AnsibleAWSError, self).__init__(exception) + if not exception: + super(AnsibleAWSError, self).__init__(message) + + self.exception = exception + self.message = message + + # In places where passing more information to module.fail_json would be helpful + # store the extra info. Other plugin types have to raise the correct exception + # such as AnsibleLookupError, so can't easily consume this. + self.kwargs = kwargs or {} + + +class AnsibleBotocoreError(AnsibleAWSError): + pass diff --git a/plugins/module_utils/modules.py b/plugins/module_utils/modules.py index 39a207a993c..016b7b8c547 100644 --- a/plugins/module_utils/modules.py +++ b/plugins/module_utils/modules.py @@ -53,7 +53,6 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from functools import wraps import logging import os import re @@ -77,7 +76,7 @@ from .botocore import get_aws_connection_info from .botocore import get_aws_region from .botocore import gather_sdk_versions - +from .retries import RetryingBotoClientWrapper from .version import LooseVersion # Currently only AnsibleAWSModule. However we have a lot of Copy and Paste code @@ -215,7 +214,7 @@ def client(self, service, retry_decorator=None): region, endpoint_url, aws_connect_kwargs = get_aws_connection_info(self, boto3=True) conn = boto3_conn(self, conn_type='client', resource=service, region=region, endpoint=endpoint_url, **aws_connect_kwargs) - return conn if retry_decorator is None else _RetryingBotoClientWrapper(conn, retry_decorator) + return conn if retry_decorator is None else RetryingBotoClientWrapper(conn, retry_decorator) def resource(self, service): region, endpoint_url, aws_connect_kwargs = get_aws_connection_info(self, boto3=True) @@ -335,39 +334,6 @@ def botocore_at_least(self, desired): return LooseVersion(existing['botocore_version']) >= LooseVersion(desired) -class _RetryingBotoClientWrapper(object): - __never_wait = ( - 'get_paginator', 'can_paginate', - 'get_waiter', 'generate_presigned_url', - ) - - def __init__(self, client, retry): - self.client = client - self.retry = retry - - def _create_optional_retry_wrapper_function(self, unwrapped): - retrying_wrapper = self.retry(unwrapped) - - @wraps(unwrapped) - def deciding_wrapper(aws_retry=False, *args, **kwargs): - if aws_retry: - return retrying_wrapper(*args, **kwargs) - else: - return unwrapped(*args, **kwargs) - return deciding_wrapper - - def __getattr__(self, name): - unwrapped = getattr(self.client, name) - if name in self.__never_wait: - return unwrapped - elif callable(unwrapped): - wrapped = self._create_optional_retry_wrapper_function(unwrapped) - setattr(self, name, wrapped) - return wrapped - else: - return unwrapped - - def _aws_common_argument_spec(): """ This does not include 'region' as some AWS APIs don't require a diff --git a/plugins/module_utils/retries.py b/plugins/module_utils/retries.py index 1bd214b6bf9..cd4fdc2ac15 100644 --- a/plugins/module_utils/retries.py +++ b/plugins/module_utils/retries.py @@ -29,6 +29,8 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +from functools import wraps + try: from botocore.exceptions import ClientError HAS_BOTO3 = True @@ -76,3 +78,36 @@ def found(response_code, catch_extra_error_codes=None): retry_on.extend(catch_extra_error_codes) return response_code in retry_on + + +class RetryingBotoClientWrapper(object): + __never_wait = ( + 'get_paginator', 'can_paginate', + 'get_waiter', 'generate_presigned_url', + ) + + def __init__(self, client, retry): + self.client = client + self.retry = retry + + def _create_optional_retry_wrapper_function(self, unwrapped): + retrying_wrapper = self.retry(unwrapped) + + @wraps(unwrapped) + def deciding_wrapper(aws_retry=False, *args, **kwargs): + if aws_retry: + return retrying_wrapper(*args, **kwargs) + else: + return unwrapped(*args, **kwargs) + return deciding_wrapper + + def __getattr__(self, name): + unwrapped = getattr(self.client, name) + if name in self.__never_wait: + return unwrapped + elif callable(unwrapped): + wrapped = self._create_optional_retry_wrapper_function(unwrapped) + setattr(self, name, wrapped) + return wrapped + else: + return unwrapped diff --git a/plugins/module_utils/waiters.py b/plugins/module_utils/waiters.py index 2abf390cbcf..6d43be5331d 100644 --- a/plugins/module_utils/waiters.py +++ b/plugins/module_utils/waiters.py @@ -11,7 +11,7 @@ except ImportError: pass # caught by HAS_BOTO3 -from ansible_collections.amazon.aws.plugins.module_utils.modules import _RetryingBotoClientWrapper +from ansible_collections.amazon.aws.plugins.module_utils.retries import RetryingBotoClientWrapper ec2_data = { @@ -1256,7 +1256,7 @@ def route53_model(name): def get_waiter(client, waiter_name): - if isinstance(client, _RetryingBotoClientWrapper): + if isinstance(client, RetryingBotoClientWrapper): return get_waiter(client.client, waiter_name) try: return waiters_by_name[(client.__class__.__name__, waiter_name)](client) diff --git a/tests/unit/plugins/modules/test_cloudformation.py b/tests/unit/plugins/modules/test_cloudformation.py index f46bc1113d6..b1756d535ee 100644 --- a/tests/unit/plugins/modules/test_cloudformation.py +++ b/tests/unit/plugins/modules/test_cloudformation.py @@ -13,7 +13,7 @@ from ansible_collections.amazon.aws.tests.unit.utils.amazon_placebo_fixtures import maybe_sleep, placeboify # pylint: disable=unused-import from ansible_collections.amazon.aws.plugins.module_utils.botocore import boto_exception -from ansible_collections.amazon.aws.plugins.module_utils.modules import _RetryingBotoClientWrapper +from ansible_collections.amazon.aws.plugins.module_utils.retries import RetryingBotoClientWrapper from ansible_collections.amazon.aws.plugins.module_utils.retries import AWSRetry from ansible_collections.amazon.aws.plugins.modules import cloudformation as cfn_module @@ -84,7 +84,7 @@ def exit_json(self, *args, **kwargs): def _create_wrapped_client(placeboify): connection = placeboify.client('cloudformation') retry_decorator = AWSRetry.jittered_backoff() - wrapped_conn = _RetryingBotoClientWrapper(connection, retry_decorator) + wrapped_conn = RetryingBotoClientWrapper(connection, retry_decorator) return wrapped_conn