is_boto3_error_code() - support list of codes and make heavier use of…

… it (#154)

* Add unit tests for is_boto3_error_code

* is_boto3_error_code: Add support for passing multiple error codes

* Move various try/except tests over to using is_boto3_error_code

* ignore duplicate-except

* Update tests

* linting

* Review followup

plugins/module_utils/core.py

@@ -318,7 +318,9 @@ def is_boto3_error_code(code, e=None):
     if e is None:
         import sys
         dummy, e, dummy = sys.exc_info()
-    if isinstance(e, ClientError) and e.response['Error']['Code'] == code:
+    if not isinstance(code, list):
+        code = [code]
+    if isinstance(e, ClientError) and e.response['Error']['Code'] in code:
         return ClientError
     return type('NeverEverRaisedException', (Exception,), {})
 

plugins/module_utils/elb_utils.py

@@ -9,6 +9,7 @@
 except ImportError:
     pass
 
+from .core import is_boto3_error_code
 from .ec2 import AWSRetry
 
 
@@ -41,11 +42,8 @@ def _get_elb(connection, module, elb_name):
     try:
         load_balancer_paginator = connection.get_paginator('describe_load_balancers')
         return (load_balancer_paginator.paginate(Names=[elb_name]).build_full_result())['LoadBalancers'][0]
-    except (BotoCoreError, ClientError) as e:
-        if e.response['Error']['Code'] == 'LoadBalancerNotFound':
-            return None
-        else:
-            raise e
+    except is_boto3_error_code('LoadBalancerNotFound'):
+        return None
 
 
 def get_elb_listener(connection, module, elb_arn, listener_port):

plugins/module_utils/iam.py

@@ -7,11 +7,12 @@
 import traceback
 
 try:
-    from botocore.exceptions import ClientError, NoCredentialsError
+    import botocore
 except ImportError:
     pass
 
 from ansible.module_utils._text import to_native
+from .core import is_boto3_error_code
 
 
 def get_aws_account_id(module):
@@ -28,22 +29,17 @@ def get_aws_account_id(module):
         account_id = sts_client.get_caller_identity().get('Account')
     # non-STS sessions may also get NoCredentialsError from this STS call, so
     # we must catch that too and try the IAM version
-    except (ClientError, NoCredentialsError):
+    except (botocore.exceptions.ClientError, botocore.exceptions.NoCredentialsError):
         try:
             iam_client = module.client('iam')
             account_id = iam_client.get_user()['User']['Arn'].split(':')[4]
-        except ClientError as e:
-            if (e.response['Error']['Code'] == 'AccessDenied'):
-                except_msg = to_native(e)
-                # don't match on `arn:aws` because of China region `arn:aws-cn` and similar
-                account_id = except_msg.search(r"arn:\w+:iam::([0-9]{12,32}):\w+/").group(1)
-            if account_id is None:
-                module.fail_json_aws(e, msg="Could not get AWS account information")
-        except Exception as e:
-            module.fail_json(
-                msg="Failed to get AWS account information, Try allowing sts:GetCallerIdentity or iam:GetUser permissions.",
-                exception=traceback.format_exc()
-            )
+        except is_boto3_error_code('AccessDenied') as e:
+            except_msg = to_native(e)
+            # don't match on `arn:aws:` because of China region `arn:aws-cn` and similar
+            account_id = except_msg.search(r"arn:\w+:iam::([0-9]{12,32}):\w+/").group(1)
+        except (LookupError, botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:  # pylint: disable=duplicate-except
+            account_id = None
+
     if not account_id:
         module.fail_json(msg="Failed while determining AWS account ID. Try allowing sts:GetCallerIdentity or iam:GetUser permissions.")
     return to_native(account_id)

plugins/modules/aws_s3.py

@@ -288,6 +288,7 @@
 
 from ..module_utils.core import AnsibleAWSModule
 from ..module_utils.core import is_boto3_error_code
+from ..module_utils.core import is_boto3_error_message
 from ..module_utils.ec2 import AWSRetry
 from ..module_utils.ec2 import boto3_conn
 from ..module_utils.ec2 import get_aws_connection_info
@@ -302,25 +303,20 @@ class Sigv4Required(Exception):
 
 
 def key_check(module, s3, bucket, obj, version=None, validate=True):
-    exists = True
     try:
         if version:
             s3.head_object(Bucket=bucket, Key=obj, VersionId=version)
         else:
             s3.head_object(Bucket=bucket, Key=obj)
-    except botocore.exceptions.ClientError as e:
-        # if a client error is thrown, check if it's a 404 error
-        # if it's a 404 error, then the object does not exist
-        error_code = int(e.response['Error']['Code'])
-        if error_code == 404:
-            exists = False
-        elif error_code == 403 and validate is False:
-            pass
-        else:
+    except is_boto3_error_code('404'):
+        return False
+    except is_boto3_error_code('403') as e:
+        if validate is True:
             module.fail_json_aws(e, msg="Failed while looking up object (during key check) %s." % obj)
-    except botocore.exceptions.BotoCoreError as e:
+    except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
         module.fail_json_aws(e, msg="Failed while looking up object (during key check) %s." % obj)
-    return exists
+
+    return True
 
 
 def etag_compare(module, local_file, s3, bucket, obj, version=None):
@@ -344,19 +340,14 @@ def bucket_check(module, s3, bucket, validate=True):
     exists = True
     try:
         s3.head_bucket(Bucket=bucket)
-    except botocore.exceptions.ClientError as e:
-        # If a client error is thrown, then check that it was a 404 error.
-        # If it was a 404 error, then the bucket does not exist.
-        error_code = int(e.response['Error']['Code'])
-        if error_code == 404:
-            exists = False
-        elif error_code == 403 and validate is False:
-            pass
-        else:
+    except is_boto3_error_code('404'):
+        return False
+    except is_boto3_error_code('403') as e:
+        if validate is True:
             module.fail_json_aws(e, msg="Failed while looking up bucket (during bucket_check) %s." % bucket)
     except botocore.exceptions.EndpointConnectionError as e:
         module.fail_json_aws(e, msg="Invalid endpoint provided")
-    except botocore.exceptions.BotoCoreError as e:
+    except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
         module.fail_json_aws(e, msg="Failed while looking up bucket (during bucket_check) %s." % bucket)
     return exists
 
@@ -379,12 +370,9 @@ def create_bucket(module, s3, bucket, location=None):
             AWSRetry.jittered_backoff(
                 max_delay=120, catch_extra_error_codes=['NoSuchBucket']
             )(s3.put_bucket_acl)(ACL=acl, Bucket=bucket)
-    except botocore.exceptions.ClientError as e:
-        if e.response['Error']['Code'] in IGNORE_S3_DROP_IN_EXCEPTIONS:
-            module.warn("PutBucketAcl is not implemented by your storage provider. Set the permission parameters to the empty list to avoid this warning")
-        else:
-            module.fail_json_aws(e, msg="Failed while creating bucket or setting acl (check that you have CreateBucket and PutBucketAcl permission).")
-    except botocore.exceptions.BotoCoreError as e:
+    except is_boto3_error_code(IGNORE_S3_DROP_IN_EXCEPTIONS):
+        module.warn("PutBucketAcl is not implemented by your storage provider. Set the permission parameters to the empty list to avoid this warning")
+    except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
         module.fail_json_aws(e, msg="Failed while creating bucket or setting acl (check that you have CreateBucket and PutBucketAcl permission).")
 
     if bucket:
@@ -404,10 +392,9 @@ def paginated_versioned_list_with_fallback(s3, **pagination_params):
             delete_markers = [{'Key': data['Key'], 'VersionId': data['VersionId']} for data in page.get('DeleteMarkers', [])]
             current_objects = [{'Key': data['Key'], 'VersionId': data['VersionId']} for data in page.get('Versions', [])]
             yield delete_markers + current_objects
-    except botocore.exceptions.ClientError as e:
-        if to_text(e.response['Error']['Code']) in IGNORE_S3_DROP_IN_EXCEPTIONS + ['AccessDenied']:
-            for page in paginated_list(s3, **pagination_params):
-                yield [{'Key': data['Key']} for data in page]
+    except is_boto3_error_code(IGNORE_S3_DROP_IN_EXCEPTIONS + ['AccessDenied']):
+        for page in paginated_list(s3, **pagination_params):
+            yield [{'Key': data['Key']} for data in page]
 
 
 def list_keys(module, s3, bucket, prefix, marker, max_keys):
@@ -463,12 +450,9 @@ def create_dirkey(module, s3, bucket, obj, encrypt):
         s3.put_object(**params)
         for acl in module.params.get('permission'):
             s3.put_object_acl(ACL=acl, Bucket=bucket, Key=obj)
-    except botocore.exceptions.ClientError as e:
-        if e.response['Error']['Code'] in IGNORE_S3_DROP_IN_EXCEPTIONS:
-            module.warn("PutObjectAcl is not implemented by your storage provider. Set the permissions parameters to the empty list to avoid this warning")
-        else:
-            module.fail_json_aws(e, msg="Failed while creating object %s." % obj)
-    except botocore.exceptions.BotoCoreError as e:
+    except is_boto3_error_code(IGNORE_S3_DROP_IN_EXCEPTIONS):
+        module.warn("PutObjectAcl is not implemented by your storage provider. Set the permissions parameters to the empty list to avoid this warning")
+    except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
         module.fail_json_aws(e, msg="Failed while creating object %s." % obj)
     module.exit_json(msg="Virtual directory %s created in bucket %s" % (obj, bucket), changed=True)
 
@@ -528,12 +512,9 @@ def upload_s3file(module, s3, bucket, obj, src, expiry, metadata, encrypt, heade
     try:
         for acl in module.params.get('permission'):
             s3.put_object_acl(ACL=acl, Bucket=bucket, Key=obj)
-    except botocore.exceptions.ClientError as e:
-        if e.response['Error']['Code'] in IGNORE_S3_DROP_IN_EXCEPTIONS:
-            module.warn("PutObjectAcl is not implemented by your storage provider. Set the permission parameters to the empty list to avoid this warning")
-        else:
-            module.fail_json_aws(e, msg="Unable to set object ACL")
-    except botocore.exceptions.BotoCoreError as e:
+    except is_boto3_error_code(IGNORE_S3_DROP_IN_EXCEPTIONS):
+        module.warn("PutObjectAcl is not implemented by your storage provider. Set the permission parameters to the empty list to avoid this warning")
+    except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
         module.fail_json_aws(e, msg="Unable to set object ACL")
     try:
         url = s3.generate_presigned_url(ClientMethod='put_object',
@@ -554,14 +535,15 @@ def download_s3file(module, s3, bucket, obj, dest, retries, version=None):
             key = s3.get_object(Bucket=bucket, Key=obj, VersionId=version)
         else:
             key = s3.get_object(Bucket=bucket, Key=obj)
-    except botocore.exceptions.ClientError as e:
-        if e.response['Error']['Code'] == 'InvalidArgument' and 'require AWS Signature Version 4' in to_text(e):
-            raise Sigv4Required()
-        elif e.response['Error']['Code'] not in ("403", "404"):
-            # AccessDenied errors may be triggered if 1) file does not exist or 2) file exists but
-            # user does not have the s3:GetObject permission. 404 errors are handled by download_file().
-            module.fail_json_aws(e, msg="Could not find the key %s." % obj)
-    except botocore.exceptions.BotoCoreError as e:
+    except is_boto3_error_code(['404', '403']) as e:
+        # AccessDenied errors may be triggered if 1) file does not exist or 2) file exists but
+        # user does not have the s3:GetObject permission. 404 errors are handled by download_file().
+        module.fail_json_aws(e, msg="Could not find the key %s." % obj)
+    except is_boto3_error_message('require AWS Signature Version 4'):
+        raise Sigv4Required()
+    except is_boto3_error_code('InvalidArgument') as e:  # pylint: disable=duplicate-except
+        module.fail_json_aws(e, msg="Could not find the key %s." % obj)
+    except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e, msg="Could not find the key %s." % obj)
 
     optional_kwargs = {'ExtraArgs': {'VersionId': version}} if version else {}
@@ -590,12 +572,11 @@ def download_s3str(module, s3, bucket, obj, version=None, validate=True):
         else:
             contents = to_native(s3.get_object(Bucket=bucket, Key=obj)["Body"].read())
         module.exit_json(msg="GET operation complete", contents=contents, changed=True)
-    except botocore.exceptions.ClientError as e:
-        if e.response['Error']['Code'] == 'InvalidArgument' and 'require AWS Signature Version 4' in to_text(e):
-            raise Sigv4Required()
-        else:
-            module.fail_json_aws(e, msg="Failed while getting contents of object %s as a string." % obj)
-    except botocore.exceptions.BotoCoreError as e:
+    except is_boto3_error_message('require AWS Signature Version 4'):
+        raise Sigv4Required()
+    except is_boto3_error_code('InvalidArgument') as e:  # pylint: disable=duplicate-except
+        module.fail_json_aws(e, msg="Failed while getting contents of object %s as a string." % obj)
+    except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e, msg="Failed while getting contents of object %s as a string." % obj)
 
 

plugins/modules/ec2_ami.py

@@ -368,6 +368,7 @@
 from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
 
 from ..module_utils.core import AnsibleAWSModule
+from ..module_utils.core import is_boto3_error_code
 from ..module_utils.ec2 import ansible_dict_to_boto3_tag_list
 from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict
 from ..module_utils.ec2 import compare_aws_tags
@@ -570,10 +571,11 @@ def deregister_image(module, connection):
         try:
             for snapshot_id in snapshots:
                 connection.delete_snapshot(SnapshotId=snapshot_id)
-        except botocore.exceptions.ClientError as e:
-            # Don't error out if root volume snapshot was already deregistered as part of deregister_image
-            if e.response['Error']['Code'] == 'InvalidSnapshot.NotFound':
-                pass
+        # Don't error out if root volume snapshot was already deregistered as part of deregister_image
+        except is_boto3_error_code('InvalidSnapshot.NotFound'):
+            pass
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+            module.fail_json_aws(e, msg='Failed to delete snapshot.')
         exit_params['snapshots_deleted'] = snapshots
 
     module.exit_json(**exit_params)
@@ -660,10 +662,9 @@ def get_image_by_id(module, connection, image_id):
             try:
                 result['LaunchPermissions'] = connection.describe_image_attribute(Attribute='launchPermission', ImageId=image_id)['LaunchPermissions']
                 result['ProductCodes'] = connection.describe_image_attribute(Attribute='productCodes', ImageId=image_id)['ProductCodes']
-            except botocore.exceptions.ClientError as e:
-                if e.response['Error']['Code'] != 'InvalidAMIID.Unavailable':
-                    module.fail_json_aws(e, msg="Error retrieving image attributes for image %s" % image_id)
-            except botocore.exceptions.BotoCoreError as e:
+            except is_boto3_error_code('InvalidAMIID.Unavailable'):
+                pass
+            except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
                 module.fail_json_aws(e, msg="Error retrieving image attributes for image %s" % image_id)
             return result
         module.fail_json(msg="Invalid number of instances (%s) found for image_id: %s." % (str(len(images)), image_id))

plugins/modules/ec2_key.py

@@ -127,13 +127,14 @@
 import uuid
 
 try:
-    from botocore.exceptions import ClientError
+    import botocore
 except ImportError:
     pass  # caught by AnsibleAWSModule
 
 from ansible.module_utils._text import to_bytes
 
 from ..module_utils.core import AnsibleAWSModule
+from ..module_utils.core import is_boto3_error_code
 
 
 def extract_key_data(key):
@@ -170,9 +171,9 @@ def find_key_pair(module, ec2_client, name):
 
     try:
         key = ec2_client.describe_key_pairs(KeyNames=[name])['KeyPairs'][0]
-    except ClientError as err:
-        if err.response['Error']['Code'] == "InvalidKeyPair.NotFound":
-            return None
+    except is_boto3_error_code('InvalidKeyPair.NotFound'):
+        return None
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as err:
         module.fail_json_aws(err, msg="error finding keypair")
     except IndexError:
         key = None
@@ -205,7 +206,7 @@ def create_key_pair(module, ec2_client, name, key_material, force):
             else:
                 try:
                     key = ec2_client.create_key_pair(KeyName=name)
-                except ClientError as err:
+                except botocore.exceptions.ClientError as err:
                     module.fail_json_aws(err, msg="error creating key")
             key_data = extract_key_data(key)
         module.exit_json(changed=True, key=key_data, msg="key pair created")
@@ -215,7 +216,7 @@ def import_key_pair(module, ec2_client, name, key_material):
 
     try:
         key = ec2_client.import_key_pair(KeyName=name, PublicKeyMaterial=to_bytes(key_material))
-    except ClientError as err:
+    except botocore.exceptions.ClientError as err:
         module.fail_json_aws(err, msg="error importing key")
     return key
 
@@ -227,7 +228,7 @@ def delete_key_pair(module, ec2_client, name, finish_task=True):
         if not module.check_mode:
             try:
                 ec2_client.delete_key_pair(KeyName=name)
-            except ClientError as err:
+            except botocore.exceptions.ClientError as err:
                 module.fail_json_aws(err, msg="error deleting key")
         if not finish_task:
             return