Ansible - Remove output only from response

lib/ansible/module_utils/gcp_utils.py

@@ -237,12 +237,16 @@ def _merge_dictionaries(self, a, b):
         return new
 
 
-# This class takes in two dictionaries `a` and `b`.
-# These are dictionaries of arbitrary depth, but made up of standard Python
-# types only.
-# This differ will compare all values in `a` to those in `b`.
-# Note: Only keys in `a` will be compared. Extra keys in `b` will be ignored.
-# Note: On all lists, order does matter.
+# This class does difference checking according to a set of GCP-specific rules.
+# This will be primarily used for checking dictionaries.
+# In an equivalence check, the left-hand dictionary will be the request and
+# the right-hand side will be the response.
+
+# Rules:
+# Extra keys in response will be ignored.
+# Ordering of lists does not matter.
+#   - exception: lists of dictionaries are
+#     assumed to be in sorted order.
 class GcpRequest(object):
     def __init__(self, request):
         self.request = request
@@ -253,31 +257,48 @@ def __eq__(self, other):
     def __ne__(self, other):
         return not self.__eq__(other)
 
-    # Returns the difference between `self.request` and `b`
-    def difference(self, b):
-        return self._compare_dicts(self.request, b.request)
+    # Returns the difference between a request + response.
+    # While this is used under the hood for __eq__ and __ne__,
+    # it is useful for debugging.
+    def difference(self, response):
+        return self._compare_value(self.request, response.request)
 
-    def _compare_dicts(self, dict1, dict2):
+    def _compare_dicts(self, req_dict, resp_dict):
         difference = {}
-        for key in dict1:
-            difference[key] = self._compare_value(dict1.get(key), dict2.get(key))
+        for key in req_dict:
+            if resp_dict.get(key):
+                difference[key] = self._compare_value(req_dict.get(key), resp_dict.get(key))
 
         # Remove all empty values from difference.
-        difference2 = {}
+        sanitized_difference = {}
         for key in difference:
             if difference[key]:
-                difference2[key] = difference[key]
+                sanitized_difference[key] = difference[key]
 
-        return difference2
+        return sanitized_difference
 
     # Takes in two lists and compares them.
-    def _compare_lists(self, list1, list2):
+    # All things in the list should be identical (even if a dictionary)
+    def _compare_lists(self, req_list, resp_list):
+        # Have to convert each thing over to unicode.
+        # Python doesn't handle equality checks between unicode + non-unicode well.
         difference = []
-        for index in range(len(list1)):
-            value1 = list1[index]
-            if index < len(list2):
-                value2 = list2[index]
-                difference.append(self._compare_value(value1, value2))
+        new_req_list = self._convert_value(req_list)
+        new_resp_list = self._convert_value(resp_list)
+
+        # We have to compare each thing in the request to every other thing
+        # in the response.
+        # This is because the request value will be a subset of the response value.
+        # The assumption is that these lists will be small enough that it won't
+        # be a performance burden.
+        for req_item in new_req_list:
+            found_item = False
+            for resp_item in new_resp_list:
+                # Looking for a None value here.
+                if not self._compare_value(req_item, resp_item):
+                    found_item = True
+            if not found_item:
+                difference.append(req_item)
 
         difference2 = []
         for value in difference:
@@ -286,49 +307,69 @@ def _compare_lists(self, list1, list2):
 
         return difference2
 
-    def _compare_value(self, value1, value2):
+    # Compare two values of arbitrary types.
+    def _compare_value(self, req_value, resp_value):
         diff = None
         # If a None is found, a difference does not exist.
         # Only differing values matter.
-        if not value2:
+        if not resp_value:
             return None
 
         # Can assume non-None types at this point.
         try:
-            if isinstance(value1, list):
-                diff = self._compare_lists(value1, value2)
-            elif isinstance(value2, dict):
-                diff = self._compare_dicts(value1, value2)
-            elif isinstance(value1, bool):
-                diff = self._compare_boolean(value1, value2)
+            if isinstance(req_value, list):
+                diff = self._compare_lists(req_value, resp_value)
+            elif isinstance(req_value, dict):
+                diff = self._compare_dicts(req_value, resp_value)
+            elif isinstance(req_value, bool):
+                diff = self._compare_boolean(req_value, resp_value)
             # Always use to_text values to avoid unicode issues.
-            elif to_text(value1) != to_text(value2):
-                diff = value1
+            elif to_text(req_value) != to_text(resp_value):
+                diff = req_value
         # to_text may throw UnicodeErrors.
         # These errors shouldn't crash Ansible and should be hidden.
         except UnicodeError:
             pass
 
         return diff
 
-    def _compare_boolean(self, value1, value2):
+    # Compare two boolean values.
+    def _compare_boolean(self, req_value, resp_value):
         try:
             # Both True
-            if value1 and isinstance(value2, bool) and value2:
+            if req_value and isinstance(resp_value, bool) and resp_value:
                 return None
-            # Value1 True, value2 'true'
-            elif value1 and to_text(value2) == 'true':
+            # Value1 True, resp_value 'true'
+            elif req_value and to_text(resp_value) == 'true':
                 return None
             # Both False
-            elif not value1 and isinstance(value2, bool) and not value2:
+            elif not req_value and isinstance(resp_value, bool) and not resp_value:
                 return None
-            # Value1 False, value2 'false'
-            elif not value1 and to_text(value2) == 'false':
+            # Value1 False, resp_value 'false'
+            elif not req_value and to_text(resp_value) == 'false':
                 return None
             else:
-                return value2
+                return resp_value
 
         # to_text may throw UnicodeErrors.
         # These errors shouldn't crash Ansible and should be hidden.
         except UnicodeError:
             return None
+
+    # Python (2 esp.) doesn't do comparisons between unicode + non-unicode well.
+    # This leads to a lot of false positives when diffing values.
+    # The Ansible to_text() function is meant to get all strings
+    # into a standard format.
+    def _convert_value(self, value):
+        if isinstance(value, list):
+            new_list = []
+            for item in value:
+                new_list.append(self._convert_value(item))
+            return new_list
+        elif isinstance(value, dict):
+            new_dict = {}
+            for key in value:
+                new_dict[key] = self._convert_value(value[key])
+            return new_dict
+        else:
+            return to_text(value)

lib/ansible/modules/cloud/google/gcp_bigquery_table.py

@@ -1254,22 +1254,10 @@ def __init__(self, request, module):
             self.request = {}
 
     def to_request(self):
-        return remove_nones_from_dict(
-            {
-                u'estimatedBytes': self.request.get('estimated_bytes'),
-                u'estimatedRows': self.request.get('estimated_rows'),
-                u'oldestEntryTime': self.request.get('oldest_entry_time'),
-            }
-        )
+        return remove_nones_from_dict({})
 
     def from_response(self):
-        return remove_nones_from_dict(
-            {
-                u'estimatedBytes': self.request.get(u'estimatedBytes'),
-                u'estimatedRows': self.request.get(u'estimatedRows'),
-                u'oldestEntryTime': self.request.get(u'oldestEntryTime'),
-            }
-        )
+        return remove_nones_from_dict({})
 
 
 class TableSchema(object):

lib/ansible/modules/cloud/google/gcp_compute_backend_service.py

@@ -981,7 +981,6 @@ def to_request(self):
                 u'enabled': self.request.get('enabled'),
                 u'oauth2ClientId': self.request.get('oauth2_client_id'),
                 u'oauth2ClientSecret': self.request.get('oauth2_client_secret'),
-                u'oauth2ClientSecretSha256': self.request.get('oauth2_client_secret_sha256'),
             }
         )
 
@@ -991,7 +990,6 @@ def from_response(self):
                 u'enabled': self.request.get(u'enabled'),
                 u'oauth2ClientId': self.request.get(u'oauth2ClientId'),
                 u'oauth2ClientSecret': self.request.get(u'oauth2ClientSecret'),
-                u'oauth2ClientSecretSha256': self.request.get(u'oauth2ClientSecretSha256'),
             }
         )
 

lib/ansible/modules/cloud/google/gcp_compute_disk.py

@@ -664,14 +664,10 @@ def __init__(self, request, module):
             self.request = {}
 
     def to_request(self):
-        return remove_nones_from_dict(
-            {u'rawKey': self.request.get('raw_key'), u'sha256': self.request.get('sha256'), u'kmsKeyName': self.request.get('kms_key_name')}
-        )
+        return remove_nones_from_dict({u'rawKey': self.request.get('raw_key'), u'kmsKeyName': self.request.get('kms_key_name')})
 
     def from_response(self):
-        return remove_nones_from_dict(
-            {u'rawKey': self.request.get(u'rawKey'), u'sha256': self.request.get(u'sha256'), u'kmsKeyName': self.request.get(u'kmsKeyName')}
-        )
+        return remove_nones_from_dict({u'rawKey': self.request.get(u'rawKey'), u'kmsKeyName': self.request.get(u'kmsKeyName')})
 
 
 class DiskDiskencryptionkey(object):
@@ -683,14 +679,10 @@ def __init__(self, request, module):
             self.request = {}
 
     def to_request(self):
-        return remove_nones_from_dict(
-            {u'rawKey': self.request.get('raw_key'), u'sha256': self.request.get('sha256'), u'kmsKeyName': self.request.get('kms_key_name')}
-        )
+        return remove_nones_from_dict({u'rawKey': self.request.get('raw_key'), u'kmsKeyName': self.request.get('kms_key_name')})
 
     def from_response(self):
-        return remove_nones_from_dict(
-            {u'rawKey': self.request.get(u'rawKey'), u'sha256': self.request.get(u'sha256'), u'kmsKeyName': self.request.get(u'kmsKeyName')}
-        )
+        return remove_nones_from_dict({u'rawKey': self.request.get(u'rawKey'), u'kmsKeyName': self.request.get(u'kmsKeyName')})
 
 
 class DiskSourcesnapshotencryptionkey(object):
@@ -702,14 +694,10 @@ def __init__(self, request, module):
             self.request = {}
 
     def to_request(self):
-        return remove_nones_from_dict(
-            {u'rawKey': self.request.get('raw_key'), u'kmsKeyName': self.request.get('kms_key_name'), u'sha256': self.request.get('sha256')}
-        )
+        return remove_nones_from_dict({u'rawKey': self.request.get('raw_key'), u'kmsKeyName': self.request.get('kms_key_name')})
 
     def from_response(self):
-        return remove_nones_from_dict(
-            {u'rawKey': self.request.get(u'rawKey'), u'kmsKeyName': self.request.get(u'kmsKeyName'), u'sha256': self.request.get(u'sha256')}
-        )
+        return remove_nones_from_dict({u'rawKey': self.request.get(u'rawKey'), u'kmsKeyName': self.request.get(u'kmsKeyName')})
 
 
 if __name__ == '__main__':

lib/ansible/modules/cloud/google/gcp_compute_image.py

@@ -730,10 +730,10 @@ def __init__(self, request, module):
             self.request = {}
 
     def to_request(self):
-        return remove_nones_from_dict({u'rawKey': self.request.get('raw_key'), u'sha256': self.request.get('sha256')})
+        return remove_nones_from_dict({u'rawKey': self.request.get('raw_key')})
 
     def from_response(self):
-        return remove_nones_from_dict({u'rawKey': self.request.get(u'rawKey'), u'sha256': self.request.get(u'sha256')})
+        return remove_nones_from_dict({u'rawKey': self.request.get(u'rawKey')})
 
 
 class ImageRawdisk(object):
@@ -764,10 +764,10 @@ def __init__(self, request, module):
             self.request = {}
 
     def to_request(self):
-        return remove_nones_from_dict({u'rawKey': self.request.get('raw_key'), u'sha256': self.request.get('sha256')})
+        return remove_nones_from_dict({u'rawKey': self.request.get('raw_key')})
 
     def from_response(self):
-        return remove_nones_from_dict({u'rawKey': self.request.get(u'rawKey'), u'sha256': self.request.get(u'sha256')})
+        return remove_nones_from_dict({u'rawKey': self.request.get(u'rawKey')})
 
 
 if __name__ == '__main__':

lib/ansible/modules/cloud/google/gcp_compute_instance.py

@@ -1298,14 +1298,10 @@ def __init__(self, request, module):
             self.request = {}
 
     def to_request(self):
-        return remove_nones_from_dict(
-            {u'rawKey': self.request.get('raw_key'), u'rsaEncryptedKey': self.request.get('rsa_encrypted_key'), u'sha256': self.request.get('sha256')}
-        )
+        return remove_nones_from_dict({u'rawKey': self.request.get('raw_key'), u'rsaEncryptedKey': self.request.get('rsa_encrypted_key')})
 
     def from_response(self):
-        return remove_nones_from_dict(
-            {u'rawKey': self.request.get(u'rawKey'), u'rsaEncryptedKey': self.request.get(u'rsaEncryptedKey'), u'sha256': self.request.get(u'sha256')}
-        )
+        return remove_nones_from_dict({u'rawKey': self.request.get(u'rawKey'), u'rsaEncryptedKey': self.request.get(u'rsaEncryptedKey')})
 
 
 class InstanceInitializeparams(object):
@@ -1348,10 +1344,10 @@ def __init__(self, request, module):
             self.request = {}
 
     def to_request(self):
-        return remove_nones_from_dict({u'rawKey': self.request.get('raw_key'), u'sha256': self.request.get('sha256')})
+        return remove_nones_from_dict({u'rawKey': self.request.get('raw_key')})
 
     def from_response(self):
-        return remove_nones_from_dict({u'rawKey': self.request.get(u'rawKey'), u'sha256': self.request.get(u'sha256')})
+        return remove_nones_from_dict({u'rawKey': self.request.get(u'rawKey')})
 
 
 class InstanceGuestacceleratorsArray(object):
@@ -1406,7 +1402,6 @@ def _request_for_item(self, item):
             {
                 u'accessConfigs': InstanceAccessconfigsArray(item.get('access_configs', []), self.module).to_request(),
                 u'aliasIpRanges': InstanceAliasiprangesArray(item.get('alias_ip_ranges', []), self.module).to_request(),
-                u'name': item.get('name'),
                 u'network': replace_resource_dict(item.get(u'network', {}), 'selfLink'),
                 u'networkIP': item.get('network_ip'),
                 u'subnetwork': replace_resource_dict(item.get(u'subnetwork', {}), 'selfLink'),
@@ -1418,7 +1413,6 @@ def _response_from_item(self, item):
             {
                 u'accessConfigs': InstanceAccessconfigsArray(item.get(u'accessConfigs', []), self.module).from_response(),
                 u'aliasIpRanges': InstanceAliasiprangesArray(item.get(u'aliasIpRanges', []), self.module).from_response(),
-                u'name': item.get(u'name'),
                 u'network': item.get(u'network'),
                 u'networkIP': item.get(u'networkIP'),
                 u'subnetwork': item.get(u'subnetwork'),

lib/ansible/modules/cloud/google/gcp_compute_instance_group_manager.py

@@ -537,32 +537,10 @@ def __init__(self, request, module):
             self.request = {}
 
     def to_request(self):
-        return remove_nones_from_dict(
-            {
-                u'abandoning': self.request.get('abandoning'),
-                u'creating': self.request.get('creating'),
-                u'creatingWithoutRetries': self.request.get('creating_without_retries'),
-                u'deleting': self.request.get('deleting'),
-                u'none': self.request.get('none'),
-                u'recreating': self.request.get('recreating'),
-                u'refreshing': self.request.get('refreshing'),
-                u'restarting': self.request.get('restarting'),
-            }
-        )
+        return remove_nones_from_dict({})
 
     def from_response(self):
-        return remove_nones_from_dict(
-            {
-                u'abandoning': self.request.get(u'abandoning'),
-                u'creating': self.request.get(u'creating'),
-                u'creatingWithoutRetries': self.request.get(u'creatingWithoutRetries'),
-                u'deleting': self.request.get(u'deleting'),
-                u'none': self.request.get(u'none'),
-                u'recreating': self.request.get(u'recreating'),
-                u'refreshing': self.request.get(u'refreshing'),
-                u'restarting': self.request.get(u'restarting'),
-            }
-        )
+        return remove_nones_from_dict({})
 
 
 class InstanceGroupManagerNamedportsArray(object):