Big Black PR (#1784)

* Black prep

* Black

* changelog

* Fix pylint unused-import in tests

* Split SSM connection plugin changes

* disable glue tests - bucket's missing

* Disable s3_logging and s3_sync tests

changelogs/fragments/1784-black.yml

@@ -0,0 +1,2 @@
+minor_changes:
+- The ``black`` code formatter has been run across the collection to improve code consistency (https://github.com/ansible-collections/community.aws/pull/1784).

plugins/module_utils/base.py

@@ -22,7 +22,7 @@
 from ansible_collections.amazon.aws.plugins.module_utils.tagging import boto3_tag_list_to_ansible_dict
 
 
-class BaseWaiterFactory():
+class BaseWaiterFactory:
     """
     A helper class used for creating additional waiters.
     Unlike the waiters available directly from botocore these waiters will
@@ -39,6 +39,7 @@ class BaseWaiterFactory():
     waiter = waiters.get_waiter('my_waiter_name')
     waiter.wait(**params)
     """
+
     module = None
     client = None
 
@@ -113,9 +114,14 @@ def _waiter_model_data(self):
 
     def _inject_ratelimit_retries(self, model):
         extra_retries = [
-            'RequestLimitExceeded', 'Unavailable', 'ServiceUnavailable',
-            'InternalFailure', 'InternalError', 'TooManyRequestsException',
-            'Throttling']
+            "RequestLimitExceeded",
+            "Unavailable",
+            "ServiceUnavailable",
+            "InternalFailure",
+            "InternalError",
+            "TooManyRequestsException",
+            "Throttling",
+        ]
 
         acceptors = []
         for error in extra_retries:
@@ -130,15 +136,15 @@ def _inject_ratelimit_retries(self, model):
     def get_waiter(self, waiter_name):
         waiters = self._model.waiter_names
         if waiter_name not in waiters:
-            self.module.fail_json(
-                'Unable to find waiter {0}.  Available_waiters: {1}'
-                .format(waiter_name, waiters))
+            self.module.fail_json("Unable to find waiter {0}.  Available_waiters: {1}".format(waiter_name, waiters))
         return botocore.waiter.create_waiter_with_client(
-            waiter_name, self._model, self.client,
+            waiter_name,
+            self._model,
+            self.client,
         )
 
 
-class Boto3Mixin():
+class Boto3Mixin:
     @staticmethod
     def aws_error_handler(description):
         r"""
@@ -176,11 +182,15 @@ def handler(_self, *args, **kwargs):
                 extra_ouput = _self._extra_error_output()
                 try:
                     return func(_self, *args, **kwargs)
-                except (botocore.exceptions.WaiterError) as e:
-                    _self.module.fail_json_aws(e, msg='Failed waiting for {DESC}'.format(DESC=description), **extra_ouput)
+                except botocore.exceptions.WaiterError as e:
+                    _self.module.fail_json_aws(
+                        e, msg="Failed waiting for {DESC}".format(DESC=description), **extra_ouput
+                    )
                 except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-                    _self.module.fail_json_aws(e, msg='Failed to {DESC}'.format(DESC=description), **extra_ouput)
+                    _self.module.fail_json_aws(e, msg="Failed to {DESC}".format(DESC=description), **extra_ouput)
+
             return handler
+
         return wrapper
 
     def _normalize_boto3_resource(self, resource, add_tags=False):
@@ -198,15 +208,15 @@ def _normalize_boto3_resource(self, resource, add_tags=False):
         if resource is None:
             return None
 
-        tags = resource.get('Tags', None)
+        tags = resource.get("Tags", None)
         if tags:
             tags = boto3_tag_list_to_ansible_dict(tags)
         elif add_tags or tags is not None:
             tags = {}
 
         normalized_resource = camel_dict_to_snake_dict(resource)
         if tags is not None:
-            normalized_resource['tags'] = tags
+            normalized_resource["tags"] = tags
         return normalized_resource
 
     def _extra_error_output(self):
@@ -261,9 +271,9 @@ def _waiter_config(self):
         params = dict()
         if self._wait_timeout:
             delay = min(5, self._wait_timeout)
-            max_attempts = (self._wait_timeout // delay)
+            max_attempts = self._wait_timeout // delay
             config = dict(Delay=delay, MaxAttempts=max_attempts)
-            params['WaiterConfig'] = config
+            params["WaiterConfig"] = config
         return params
 
     def _wait_for_deletion(self):
@@ -346,8 +356,7 @@ def _set_resource_value(self, key, value, description=None, immutable=False):
         if immutable and self.original_resource:
             if description is None:
                 description = key
-            self.module.fail_json(msg='{0} can not be updated after creation'
-                                  .format(description))
+            self.module.fail_json(msg="{0} can not be updated after creation".format(description))
         self._resource_updates[key] = value
         self.changed = True
         return True

plugins/module_utils/ec2.py

@@ -22,7 +22,7 @@ class Ec2WaiterFactory(BaseWaiterFactory):
     def __init__(self, module):
         # the AWSRetry wrapper doesn't support the wait functions (there's no
         # public call we can cleanly wrap)
-        client = module.client('ec2')
+        client = module.client("ec2")
         super(Ec2WaiterFactory, self).__init__(module, client)
 
     @property
@@ -32,30 +32,28 @@ def _waiter_model_data(self):
 
 
 class Ec2Boto3Mixin(Boto3Mixin):
-
     @AWSRetry.jittered_backoff()
     def _paginated_describe_subnets(self, **params):
-        paginator = self.client.get_paginator('describe_subnets')
+        paginator = self.client.get_paginator("describe_subnets")
         return paginator.paginate(**params).build_full_result()
 
-    @Boto3Mixin.aws_error_handler('describe subnets')
+    @Boto3Mixin.aws_error_handler("describe subnets")
     def _describe_subnets(self, **params):
         try:
             result = self._paginated_describe_subnets(**params)
-        except is_boto3_error_code('SubnetID.NotFound'):
+        except is_boto3_error_code("SubnetID.NotFound"):
             return None
-        return result.get('Subnets', None)
+        return result.get("Subnets", None)
 
 
 class BaseEc2Manager(Ec2Boto3Mixin, BaseResourceManager):
-
     resource_id = None
     TAG_RESOURCE_TYPE = None
     # This can be overridden by a subclass *if* 'Tags' isn't returned as a part of
     # the standard Resource description
     TAGS_ON_RESOURCE = True
     # If the resource supports using "TagSpecifications" on creation we can
-    TAGS_ON_CREATE = 'TagSpecifications'
+    TAGS_ON_CREATE = "TagSpecifications"
 
     def __init__(self, module, id=None):
         r"""
@@ -78,27 +76,27 @@ def _flush_update(self):
         changed |= super(BaseEc2Manager, self)._flush_update()
         return changed
 
-    @Boto3Mixin.aws_error_handler('connect to AWS')
-    def _create_client(self, client_name='ec2'):
+    @Boto3Mixin.aws_error_handler("connect to AWS")
+    def _create_client(self, client_name="ec2"):
         client = self.module.client(client_name, retry_decorator=AWSRetry.jittered_backoff())
         return client
 
-    @Boto3Mixin.aws_error_handler('set tags on resource')
+    @Boto3Mixin.aws_error_handler("set tags on resource")
     def _add_tags(self, **params):
         self.client.create_tags(aws_retry=True, **params)
         return True
 
-    @Boto3Mixin.aws_error_handler('unset tags on resource')
+    @Boto3Mixin.aws_error_handler("unset tags on resource")
     def _remove_tags(self, **params):
         self.client.delete_tags(aws_retry=True, **params)
         return True
 
     @AWSRetry.jittered_backoff()
     def _paginated_describe_tags(self, **params):
-        paginator = self.client.get_paginator('describe_tags')
+        paginator = self.client.get_paginator("describe_tags")
         return paginator.paginate(**params).build_full_result()
 
-    @Boto3Mixin.aws_error_handler('list tags on resource')
+    @Boto3Mixin.aws_error_handler("list tags on resource")
     def _describe_tags(self, id=None):
         if not id:
             id = self.resource_id
@@ -111,16 +109,16 @@ def _get_tags(self, id=None):
             id = self.resource_id
         # If the Tags are available from the resource, then use them
         if self.TAGS_ON_RESOURCE:
-            tags = self._preupdate_resource.get('Tags', [])
+            tags = self._preupdate_resource.get("Tags", [])
         # Otherwise we'll have to look them up
         else:
             tags = self._describe_tags(id=id)
         return boto3_tag_list_to_ansible_dict(tags)
 
     def _do_tagging(self):
         changed = False
-        tags_to_add = self._tagging_updates.get('add')
-        tags_to_remove = self._tagging_updates.get('remove')
+        tags_to_add = self._tagging_updates.get("add")
+        tags_to_remove = self._tagging_updates.get("remove")
 
         if tags_to_add:
             changed = True
@@ -136,25 +134,22 @@ def _do_tagging(self):
         return changed
 
     def _merge_resource_changes(self, filter_immutable=True, creation=False):
-
         resource = super(BaseEc2Manager, self)._merge_resource_changes(
-            filter_immutable=filter_immutable,
-            creation=creation
+            filter_immutable=filter_immutable, creation=creation
         )
 
         if creation:
             if not self.TAGS_ON_CREATE:
-                resource.pop('Tags', None)
-            elif self.TAGS_ON_CREATE == 'TagSpecifications':
-                tags = boto3_tag_list_to_ansible_dict(resource.pop('Tags', []))
+                resource.pop("Tags", None)
+            elif self.TAGS_ON_CREATE == "TagSpecifications":
+                tags = boto3_tag_list_to_ansible_dict(resource.pop("Tags", []))
                 tag_specs = boto3_tag_specifications(tags, types=[self.TAG_RESOURCE_TYPE])
                 if tag_specs:
-                    resource['TagSpecifications'] = tag_specs
+                    resource["TagSpecifications"] = tag_specs
 
         return resource
 
     def set_tags(self, tags, purge_tags):
-
         if tags is None:
             return False
         changed = False
@@ -173,16 +168,16 @@ def set_tags(self, tags, purge_tags):
         tags_to_add, tags_to_remove = compare_aws_tags(current_tags, tags, purge_tags)
 
         if tags_to_add:
-            self._tagging_updates['add'] = tags_to_add
+            self._tagging_updates["add"] = tags_to_add
             changed = True
         if tags_to_remove:
-            self._tagging_updates['remove'] = tags_to_remove
+            self._tagging_updates["remove"] = tags_to_remove
             changed = True
 
         if changed:
             # Tags are a stored as a list, but treated like a list, the
             # simplisic '==' in _set_resource_value doesn't do the comparison
             # properly
-            return self._set_resource_value('Tags', ansible_dict_to_boto3_tag_list(desired_tags))
+            return self._set_resource_value("Tags", ansible_dict_to_boto3_tag_list(desired_tags))
 
         return False

plugins/module_utils/etag.py

@@ -24,6 +24,7 @@
 
 try:
     from boto3.s3.transfer import TransferConfig
+
     DEFAULT_CHUNK_SIZE = TransferConfig().multipart_chunksize
 except ImportError:
     DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024
@@ -42,14 +43,13 @@ def calculate_multipart_etag(source_path, chunk_size=DEFAULT_CHUNK_SIZE):
 
     md5s = []
 
-    with open(source_path, 'rb') as fp:
+    with open(source_path, "rb") as fp:
         while True:
-
             data = fp.read(chunk_size)
 
             if not data:
                 break
-            md5 = hashlib.new('md5', usedforsecurity=False)
+            md5 = hashlib.new("md5", usedforsecurity=False)
             md5.update(data)
             md5s.append(md5)
 

plugins/module_utils/modules.py

@@ -10,9 +10,7 @@
 
 
 class AnsibleCommunityAWSModule(AnsibleAWSModule):
-
     def __init__(self, **kwargs):
-
         super(AnsibleCommunityAWSModule, self).__init__(**kwargs)
         set_collection_info(
             collection_name=COMMUNITY_AWS_COLLECTION_NAME,