Bulk migration to fail_json_aws (#361)

* Split imports and sort
* Move camel_dict_to_snake_dict imports to ansible.module_utils.common.dict_transformations
* Cleanup unused imports
* Bulk migration to fail_json_aws
* Changelog

changelogs/fragments/361-fail_json_aws.yml

@@ -0,0 +1,2 @@
+minor_changes:
+- various community.aws modules - migrate exception error message handling from fail_json to fail_json_aws (https://github.com/ansible-collections/community.aws/pull/361).

plugins/modules/aws_api_gateway.py

@@ -172,16 +172,17 @@
 '''
 
 import json
+import traceback
 
 try:
     import botocore
 except ImportError:
     pass  # Handled by AnsibleAWSModule
 
-import traceback
+from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
+
 from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict
 
 
 def main():

plugins/modules/aws_codepipeline.py

@@ -194,18 +194,18 @@
 '''
 
 import copy
-import traceback
-
-from ansible.module_utils._text import to_native
-from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule, is_boto3_error_code
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict, compare_policies
-
 
 try:
     import botocore
 except ImportError:
     pass  # caught by AnsibleAWSModule
 
+from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
+
+from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
+from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_policies
+
 
 def create_pipeline(client, name, role_arn, artifact_store, stages, version, module):
     pipeline_dict = {'name': name, 'roleArn': role_arn, 'artifactStore': artifact_store, 'stages': stages}
@@ -214,36 +214,24 @@ def create_pipeline(client, name, role_arn, artifact_store, stages, version, mod
     try:
         resp = client.create_pipeline(pipeline=pipeline_dict)
         return resp
-    except botocore.exceptions.ClientError as e:
-        module.fail_json(msg="Unable create pipeline {0}: {1}".format(name, to_native(e)),
-                         exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
-    except botocore.exceptions.BotoCoreError as e:
-        module.fail_json(msg="Unable to create pipeline {0}: {1}".format(name, to_native(e)),
-                         exception=traceback.format_exc())
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Unable create pipeline {0}".format(pipeline_dict['name']))
 
 
 def update_pipeline(client, pipeline_dict, module):
     try:
         resp = client.update_pipeline(pipeline=pipeline_dict)
         return resp
-    except botocore.exceptions.ClientError as e:
-        module.fail_json(msg="Unable update pipeline {0}: {1}".format(pipeline_dict['name'], to_native(e)),
-                         exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
-    except botocore.exceptions.BotoCoreError as e:
-        module.fail_json(msg="Unable to update pipeline {0}: {1}".format(pipeline_dict['name'], to_native(e)),
-                         exception=traceback.format_exc())
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Unable update pipeline {0}".format(pipeline_dict['name']))
 
 
 def delete_pipeline(client, name, module):
     try:
         resp = client.delete_pipeline(name=name)
         return resp
-    except botocore.exceptions.ClientError as e:
-        module.fail_json(msg="Unable delete pipeline {0}: {1}".format(name, to_native(e)),
-                         exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
-    except botocore.exceptions.BotoCoreError as e:
-        module.fail_json(msg="Unable to delete pipeline {0}: {1}".format(name, to_native(e)),
-                         exception=traceback.format_exc())
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Unable delete pipeline {0}".format(name))
 
 
 def describe_pipeline(client, name, version, module):

plugins/modules/aws_direct_connect_confirm_connection.py

@@ -61,15 +61,18 @@
 '''
 
 import traceback
-from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
-from ansible_collections.amazon.aws.plugins.module_utils.direct_connect import DirectConnectError
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import (camel_dict_to_snake_dict, AWSRetry)
 
 try:
     from botocore.exceptions import BotoCoreError, ClientError
 except ImportError:
     pass  # handled by imported AnsibleAWSModule
 
+from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
+
+from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
+from ansible_collections.amazon.aws.plugins.module_utils.direct_connect import DirectConnectError
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
+
 retry_params = {"tries": 10, "delay": 5, "backoff": 1.2, "catch_extra_error_codes": ["DirectConnectClientException"]}
 
 

plugins/modules/aws_direct_connect_connection.py

@@ -156,20 +156,21 @@
 """
 
 import traceback
-from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import (camel_dict_to_snake_dict, AWSRetry)
-from ansible_collections.amazon.aws.plugins.module_utils.direct_connect import (
-    DirectConnectError,
-    delete_connection,
-    associate_connection_and_lag,
-    disassociate_connection_and_lag,
-)
 
 try:
     from botocore.exceptions import BotoCoreError, ClientError
 except ImportError:
     pass  # handled by imported AnsibleAWSModule
 
+from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
+
+from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
+from ansible_collections.amazon.aws.plugins.module_utils.direct_connect import DirectConnectError
+from ansible_collections.amazon.aws.plugins.module_utils.direct_connect import associate_connection_and_lag
+from ansible_collections.amazon.aws.plugins.module_utils.direct_connect import delete_connection
+from ansible_collections.amazon.aws.plugins.module_utils.direct_connect import disassociate_connection_and_lag
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
+
 retry_params = {"tries": 10, "delay": 5, "backoff": 1.2, "catch_extra_error_codes": ["DirectConnectClientException"]}
 
 

plugins/modules/aws_direct_connect_gateway.py

@@ -97,25 +97,23 @@
 '''
 
 import time
-import traceback
 
 try:
     import botocore
 except ImportError:
     pass  # Handled by AnsibleAWSModule
 
-from ansible.module_utils._text import to_native
+from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
 
 from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict
 
 
 def dx_gateway_info(client, gateway_id, module):
     try:
         resp = client.describe_direct_connect_gateways(
             directConnectGatewayId=gateway_id)
     except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
-        module.fail_json(msg=to_native(e), exception=traceback.format_exc())
+        module.fail_json_aws(e, msg="Failed to fetch gateway information.")
     if resp['directConnectGateways']:
         return resp['directConnectGateways'][0]
 
@@ -142,7 +140,7 @@ def wait_for_status(client, module, gateway_id, virtual_gateway_id, status):
                 status_achieved = True
                 break
         except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
-            module.fail_json(msg=to_native(e), exception=traceback.format_exc())
+            module.fail_json_aws(e, msg="Failed while waiting for gateway association.")
 
     result = response
     return status_achieved, result
@@ -156,7 +154,7 @@ def associate_direct_connect_gateway(client, module, gateway_id):
             directConnectGatewayId=gateway_id,
             virtualGatewayId=params['virtual_gateway_id'])
     except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
-        module.fail_json(msg=to_native(e), exception=traceback.format_exc())
+        module.fail_json_aws(e, 'Failed to associate gateway')
 
     status_achieved, dxgw = wait_for_status(client, module, gateway_id, params['virtual_gateway_id'], 'associating')
     if not status_achieved:
@@ -172,7 +170,7 @@ def delete_association(client, module, gateway_id, virtual_gateway_id):
             directConnectGatewayId=gateway_id,
             virtualGatewayId=virtual_gateway_id)
     except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
-        module.fail_json(msg=to_native(e), exception=traceback.format_exc())
+        module.fail_json_aws(e, msg="Failed to delete gateway association.")
 
     status_achieved, dxgw = wait_for_status(client, module, gateway_id, virtual_gateway_id, 'disassociating')
     if not status_achieved:
@@ -191,7 +189,7 @@ def create_dx_gateway(client, module):
             directConnectGatewayName=params['name'],
             amazonSideAsn=int(params['amazon_asn']))
     except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
-        module.fail_json(msg=to_native(e), exception=traceback.format_exc())
+        module.fail_json_aws(e, msg="Failed to create direct connect gateway.")
 
     result = response
     return result
@@ -206,7 +204,7 @@ def find_dx_gateway(client, module, gateway_id=None):
         try:
             resp = client.describe_direct_connect_gateways(**params)
         except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
-            module.fail_json(msg=to_native(e), exception=traceback.format_exc())
+            module.fail_json_aws(e, msg="Failed to describe gateways")
         gateways.extend(resp['directConnectGateways'])
         if 'nextToken' in resp:
             params['nextToken'] = resp['nextToken']
@@ -233,7 +231,7 @@ def check_dxgw_association(client, module, gateway_id, virtual_gateway_id=None):
                 virtualGatewayId=virtual_gateway_id,
             )
     except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
-        module.fail_json(msg=to_native(e), exception=traceback.format_exc())
+        module.fail_json_aws(e, msg="Failed to check gateway association")
     return resp
 
 
@@ -330,7 +328,7 @@ def ensure_absent(client, module):
                 directConnectGatewayId=dx_gateway_id
             )
         except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
-            module.fail_json(msg=to_native(e), exception=traceback.format_exc())
+            module.fail_json_aws(e, msg="Failed to delete gateway")
         result = resp['directConnectGateway']
     return changed
 

plugins/modules/aws_direct_connect_link_aggregation_group.py

@@ -172,9 +172,10 @@
 except ImportError:
     pass  # Handled by AnsibleAWSModule
 
+from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
+
 from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict
 
 from ansible_collections.amazon.aws.plugins.module_utils.direct_connect import DirectConnectError
 from ansible_collections.amazon.aws.plugins.module_utils.direct_connect import delete_connection

plugins/modules/aws_direct_connect_virtual_interface.py

@@ -248,16 +248,20 @@
 '''
 
 import traceback
-from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
-from ansible_collections.amazon.aws.plugins.module_utils.direct_connect import DirectConnectError, delete_virtual_interface
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry, camel_dict_to_snake_dict
 
 try:
     from botocore.exceptions import ClientError, BotoCoreError
 except ImportError:
     # handled by AnsibleAWSModule
     pass
 
+from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
+
+from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
+from ansible_collections.amazon.aws.plugins.module_utils.direct_connect import DirectConnectError
+from ansible_collections.amazon.aws.plugins.module_utils.direct_connect import delete_virtual_interface
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
+
 
 def try_except_ClientError(failure_msg):
     '''

plugins/modules/aws_kms_info.py

@@ -215,17 +215,16 @@
 '''
 
 
-import traceback
-
 try:
     import botocore
 except ImportError:
     pass  # Handled by AnsibleAWSModule
 
+from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
+
 from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
 from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict
 
 # Caching lookup for aliases
@@ -309,9 +308,7 @@ def get_kms_tags(connection, module, key_id):
             tags.extend(tag_response['Tags'])
         except botocore.exceptions.ClientError as e:
             if e.response['Error']['Code'] != 'AccessDeniedException':
-                module.fail_json(msg="Failed to obtain key tags",
-                                 exception=traceback.format_exc(),
-                                 **camel_dict_to_snake_dict(e.response))
+                module.fail_json_aws(e, msg="Failed to obtain key tags")
             else:
                 tag_response = {}
         if tag_response.get('NextMarker'):
@@ -328,9 +325,7 @@ def get_kms_policies(connection, module, key_id):
                 policy in policies]
     except botocore.exceptions.ClientError as e:
         if e.response['Error']['Code'] != 'AccessDeniedException':
-            module.fail_json(msg="Failed to obtain key policies",
-                             exception=traceback.format_exc(),
-                             **camel_dict_to_snake_dict(e.response))
+            module.fail_json_aws(e, msg="Failed to obtain key policies")
         else:
             return []
 
@@ -360,18 +355,14 @@ def get_key_details(connection, module, key_id, tokens=None):
         tokens = []
     try:
         result = get_kms_metadata_with_backoff(connection, key_id)['KeyMetadata']
-    except botocore.exceptions.ClientError as e:
-        module.fail_json(msg="Failed to obtain key metadata",
-                         exception=traceback.format_exc(),
-                         **camel_dict_to_snake_dict(e.response))
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Failed to obtain key metadata")
     result['KeyArn'] = result.pop('Arn')
 
     try:
         aliases = get_kms_aliases_lookup(connection)
-    except botocore.exceptions.ClientError as e:
-        module.fail_json(msg="Failed to obtain aliases",
-                         exception=traceback.format_exc(),
-                         **camel_dict_to_snake_dict(e.response))
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Failed to obtain aliases")
     result['aliases'] = aliases.get(result['KeyId'], [])
 
     if result['Origin'] == 'AWS_KMS':
@@ -384,10 +375,8 @@ def get_key_details(connection, module, key_id, tokens=None):
 
     try:
         result['grants'] = get_kms_grants_with_backoff(connection, key_id, tokens=tokens)['Grants']
-    except botocore.exceptions.ClientError as e:
-        module.fail_json(msg="Failed to obtain key grants",
-                         exception=traceback.format_exc(),
-                         **camel_dict_to_snake_dict(e.response))
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Failed to obtain key grants")
     tags = get_kms_tags(connection, module, key_id)
 
     result = camel_dict_to_snake_dict(result)
@@ -399,10 +388,8 @@ def get_key_details(connection, module, key_id, tokens=None):
 def get_kms_info(connection, module):
     try:
         keys = get_kms_keys_with_backoff(connection)['Keys']
-    except botocore.exceptions.ClientError as e:
-        module.fail_json(msg="Failed to obtain keys",
-                         exception=traceback.format_exc(),
-                         **camel_dict_to_snake_dict(e.response))
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Failed to obtain keys")
 
     return [get_key_details(connection, module, key['KeyId']) for key in keys]
 

plugins/modules/aws_s3_bucket_info.py

@@ -49,17 +49,14 @@
   type: list
 '''
 
-import traceback
-
 try:
     import botocore
 except ImportError:
     pass  # Handled by AnsibleAWSModule
 
-from ansible.module_utils._text import to_native
+from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
 
 from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict
 
 
 def get_bucket_list(module, connection):
@@ -71,8 +68,8 @@ def get_bucket_list(module, connection):
     """
     try:
         buckets = camel_dict_to_snake_dict(connection.list_buckets())['buckets']
-    except botocore.exceptions.ClientError as e:
-        module.fail_json(msg=to_native(e), exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Failed to list buckets")
 
     return buckets