Normalize refactor (#2006) (#2027)

[PR #2006/3974a823 backport][stable-7] Normalize refactor

This is a backport of PR #2006 as merged into main (3974a82).
SUMMARY
fixes: #2001
ISSUE TYPE

Feature Pull Request

COMPONENT NAME
plugins/module_utils/iam.py
plugins/module_utils/transformation.py
ADDITIONAL INFORMATION
Note: also tinkers a little with Type Hinting, not sure of the value
TODO:

 Initial transform code
 Unit tests for transform code
 initial iam updates
 Basic unit tests for iam updates
 changelog

Reviewed-by: Helen Bailey <[email protected]>

changelogs/fragments/2006-normalize-refactor.yml

@@ -0,0 +1,3 @@
+minor_changes:
+- module_utils.transformations - add ``boto3_resource_to_ansible_dict()`` and ``boto3_resource_list_to_ansible_dict()`` helpers (https://github.com/ansible-collections/amazon.aws/pull/2006).
+- module_utils.iam - refactored normalization functions to use ``boto3_resource_to_ansible_dict()`` and ``boto3_resource_list_to_ansible_dict()`` (https://github.com/ansible-collections/amazon.aws/pull/2006).

plugins/module_utils/iam.py

@@ -4,25 +4,27 @@
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
 import re
-from copy import deepcopy
 
 try:
     import botocore
 except ImportError:
     pass  # Modules are responsible for handling this.
 
 from ansible.module_utils._text import to_native
-from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
 
 from .arn import parse_aws_arn
 from .arn import validate_aws_arn
 from .botocore import is_boto3_error_code
-from .botocore import normalize_boto3_result
 from .errors import AWSErrorHandler
 from .exceptions import AnsibleAWSError
 from .retries import AWSRetry
 from .tagging import ansible_dict_to_boto3_tag_list
-from .tagging import boto3_tag_list_to_ansible_dict
+from .transformation import AnsibleAWSResource
+from .transformation import AnsibleAWSResourceList
+from .transformation import BotoResource
+from .transformation import BotoResourceList
+from .transformation import boto3_resource_list_to_ansible_dict
+from .transformation import boto3_resource_to_ansible_dict
 
 
 class AnsibleIAMError(AnsibleAWSError):
@@ -198,66 +200,6 @@ def get_iam_managed_policy_version(client, arn, version):
     return client.get_policy_version(PolicyArn=arn, VersionId=version)["PolicyVersion"]
 
 
-def normalize_iam_mfa_device(device):
-    """Converts IAM MFA Device from the CamelCase boto3 format to the snake_case Ansible format"""
-    if not device:
-        return device
-    camel_device = camel_dict_to_snake_dict(device)
-    camel_device["tags"] = boto3_tag_list_to_ansible_dict(device.pop("Tags", []))
-    return camel_device
-
-
-def normalize_iam_mfa_devices(devices):
-    """Converts a list of IAM MFA Devices from the CamelCase boto3 format to the snake_case Ansible format"""
-    if not devices:
-        return []
-    devices = [normalize_iam_mfa_device(d) for d in devices]
-    return devices
-
-
-def normalize_iam_user(user):
-    """Converts IAM users from the CamelCase boto3 format to the snake_case Ansible format"""
-    if not user:
-        return user
-    camel_user = camel_dict_to_snake_dict(user)
-    camel_user["tags"] = boto3_tag_list_to_ansible_dict(user.pop("Tags", []))
-    return camel_user
-
-
-def normalize_iam_policy(policy):
-    """Converts IAM policies from the CamelCase boto3 format to the snake_case Ansible format"""
-    if not policy:
-        return policy
-    camel_policy = camel_dict_to_snake_dict(policy)
-    camel_policy["tags"] = boto3_tag_list_to_ansible_dict(policy.get("Tags", []))
-    return camel_policy
-
-
-def normalize_iam_group(group):
-    """Converts IAM Groups from the CamelCase boto3 format to the snake_case Ansible format"""
-    if not group:
-        return group
-    camel_group = camel_dict_to_snake_dict(normalize_boto3_result(group))
-    return camel_group
-
-
-def normalize_iam_access_key(access_key):
-    """Converts IAM access keys from the CamelCase boto3 format to the snake_case Ansible format"""
-    if not access_key:
-        return access_key
-    camel_key = camel_dict_to_snake_dict(normalize_boto3_result(access_key))
-    return camel_key
-
-
-def normalize_iam_access_keys(access_keys):
-    """Converts a list of IAM access keys from the CamelCase boto3 format to the snake_case Ansible format"""
-    if not access_keys:
-        return []
-    access_keys = [normalize_iam_access_key(k) for k in access_keys]
-    sorted_keys = sorted(access_keys, key=lambda d: d.get("create_date", None))
-    return sorted_keys
-
-
 def convert_managed_policy_names_to_arns(client, policy_names):
     if all(validate_aws_arn(policy, service="iam") for policy in policy_names if policy is not None):
         return policy_names
@@ -386,47 +328,6 @@ def list_iam_instance_profiles(client, name=None, prefix=None, role=None):
     return _list_iam_instance_profiles(client)
 
 
-def normalize_iam_instance_profile(profile, _v7_compat=False):
-    """
-    Converts a boto3 format IAM instance profile into "Ansible" format
-
-    _v7_compat is deprecated and will be removed in release after 2025-05-01 DO NOT USE.
-    """
-
-    new_profile = camel_dict_to_snake_dict(deepcopy(profile))
-    if profile.get("Roles"):
-        new_profile["roles"] = [normalize_iam_role(role, _v7_compat=_v7_compat) for role in profile.get("Roles")]
-    if profile.get("Tags"):
-        new_profile["tags"] = boto3_tag_list_to_ansible_dict(profile.get("Tags"))
-    else:
-        new_profile["tags"] = {}
-    new_profile["original"] = profile
-    return new_profile
-
-
-def normalize_iam_role(role, _v7_compat=False):
-    """
-    Converts a boto3 format IAM instance role into "Ansible" format
-
-    _v7_compat is deprecated and will be removed in release after 2025-05-01 DO NOT USE.
-    """
-
-    new_role = camel_dict_to_snake_dict(deepcopy(role))
-    if role.get("InstanceProfiles"):
-        new_role["instance_profiles"] = [
-            normalize_iam_instance_profile(profile, _v7_compat=_v7_compat) for profile in role.get("InstanceProfiles")
-        ]
-    if role.get("AssumeRolePolicyDocument"):
-        if _v7_compat:
-            # new_role["assume_role_policy_document"] = role.get("AssumeRolePolicyDocument")
-            new_role["assume_role_policy_document_raw"] = role.get("AssumeRolePolicyDocument")
-        else:
-            new_role["assume_role_policy_document"] = role.get("AssumeRolePolicyDocument")
-
-    new_role["tags"] = boto3_tag_list_to_ansible_dict(role.get("Tags", []))
-    return new_role
-
-
 @IAMErrorHandler.common_error_handler("tag instance profile")
 @AWSRetry.jittered_backoff()
 def tag_iam_instance_profile(client, name, tags):
@@ -497,3 +398,83 @@ def validate_iam_identifiers(resource_type, name=None, path=None):
         return path_problem
 
     return None
+
+
+def normalize_iam_mfa_device(device: BotoResource) -> AnsibleAWSResource:
+    """Converts IAM MFA Device from the CamelCase boto3 format to the snake_case Ansible format"""
+    # MFA Devices don't support Tags (as of 1.34.52)
+    return boto3_resource_to_ansible_dict(device)
+
+
+def normalize_iam_mfa_devices(devices: BotoResourceList) -> AnsibleAWSResourceList:
+    """Converts a list of IAM MFA Devices from the CamelCase boto3 format to the snake_case Ansible format"""
+    # MFA Devices don't support Tags (as of 1.34.52)
+    return boto3_resource_list_to_ansible_dict(devices)
+
+
+def normalize_iam_user(user: BotoResource) -> AnsibleAWSResource:
+    """Converts IAM users from the CamelCase boto3 format to the snake_case Ansible format"""
+    return boto3_resource_to_ansible_dict(user)
+
+
+def normalize_iam_policy(policy: BotoResource) -> AnsibleAWSResource:
+    """Converts IAM policies from the CamelCase boto3 format to the snake_case Ansible format"""
+    return boto3_resource_to_ansible_dict(policy)
+
+
+def normalize_iam_group(group: BotoResource) -> AnsibleAWSResource:
+    """Converts IAM Groups from the CamelCase boto3 format to the snake_case Ansible format"""
+    # Groups don't support Tags (as of 1.34.52)
+    return boto3_resource_to_ansible_dict(group, force_tags=False)
+
+
+def normalize_iam_access_key(access_key: BotoResource) -> AnsibleAWSResource:
+    """Converts IAM access keys from the CamelCase boto3 format to the snake_case Ansible format"""
+    # Access Keys don't support Tags (as of 1.34.52)
+    return boto3_resource_to_ansible_dict(access_key, force_tags=False)
+
+
+def normalize_iam_access_keys(access_keys: BotoResourceList) -> AnsibleAWSResourceList:
+    """Converts a list of IAM access keys from the CamelCase boto3 format to the snake_case Ansible format"""
+    # Access Keys don't support Tags (as of 1.34.52)
+    if not access_keys:
+        return access_keys
+    access_keys = boto3_resource_list_to_ansible_dict(access_keys, force_tags=False)
+    return sorted(access_keys, key=lambda d: d.get("create_date", None))
+
+
+def normalize_iam_instance_profile(profile: BotoResource) -> AnsibleAWSResource:
+    """
+    Converts a boto3 format IAM instance profile into "Ansible" format
+
+    _v7_compat is deprecated and will be removed in release after 2025-05-01 DO NOT USE.
+    """
+    transforms = {"Roles": _normalize_iam_roles}
+    transformed_profile = boto3_resource_to_ansible_dict(profile, nested_transforms=transforms)
+    return transformed_profile
+
+
+def normalize_iam_role(role: BotoResource, _v7_compat: bool = False) -> AnsibleAWSResource:
+    """
+    Converts a boto3 format IAM instance role into "Ansible" format
+
+    _v7_compat is deprecated and will be removed in release after 2025-05-01 DO NOT USE.
+    """
+    transforms = {"InstanceProfiles": _normalize_iam_instance_profiles}
+    ignore_list = [] if _v7_compat else ["AssumeRolePolicyDocument"]
+    transformed_role = boto3_resource_to_ansible_dict(role, nested_transforms=transforms, ignore_list=ignore_list)
+    if _v7_compat and role.get("AssumeRolePolicyDocument"):
+        transformed_role["assume_role_policy_document_raw"] = role["AssumeRolePolicyDocument"]
+    return transformed_role
+
+
+def _normalize_iam_instance_profiles(profiles: BotoResourceList) -> AnsibleAWSResourceList:
+    if not profiles:
+        return profiles
+    return [normalize_iam_instance_profile(p) for p in profiles]
+
+
+def _normalize_iam_roles(roles: BotoResourceList) -> AnsibleAWSResourceList:
+    if not roles:
+        return roles
+    return [normalize_iam_role(r) for r in roles]

plugins/module_utils/transformation.py

@@ -28,9 +28,26 @@
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+from copy import deepcopy
+from typing import Any
+from typing import Callable
+from typing import Mapping
+from typing import Optional
+from typing import Sequence
+from typing import Union
+
+from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
 from ansible.module_utils.six import integer_types
 from ansible.module_utils.six import string_types
 
+from .botocore import normalize_boto3_result
+from .tagging import boto3_tag_list_to_ansible_dict
+
+BotoResource = Union[None, Mapping[str, Any]]
+BotoResourceList = Union[None, Sequence[Mapping[str, Any]]]
+AnsibleAWSResource = Union[None, Mapping[str, Any]]
+AnsibleAWSResourceList = Union[None, Sequence[Mapping[str, Any]]]
+
 
 def ansible_dict_to_boto3_filter_list(filters_dict):
     """Convert an Ansible dict of filters to list of dicts that boto3 can use
@@ -133,3 +150,82 @@ def scrub_none_parameters(parameters, descend_into_lists=True):
             clean_parameters[k] = v
 
     return clean_parameters
+
+
+def _perform_nested_transforms(
+    resource: Mapping[str, Any],
+    nested_transforms: Optional[Mapping[str, Callable]],
+) -> Mapping[str, Any]:
+    if not nested_transforms:
+        return resource
+
+    for k, transform in nested_transforms.items():
+        if k in resource:
+            resource[k] = transform(resource[k])
+
+    return resource
+
+
+def boto3_resource_to_ansible_dict(
+    resource: BotoResource,
+    transform_tags: bool = True,
+    force_tags: bool = True,
+    normalize: bool = True,
+    ignore_list: Optional[Sequence[str]] = None,
+    nested_transforms: Optional[Mapping[str, Callable]] = None,
+) -> AnsibleAWSResource:
+    """
+    Transforms boto3-style (CamelCase) resource to the ansible-style (snake_case).
+
+    :param resource: a dictionary representing the resource
+    :param transform_tags: whether or not to perform "tag list" to "dictionary" conversion on the "Tags" key
+    :param normalize: whether resources should be passed through .botocore.normalize_boto3_result
+    :param ignore_list: a list of keys, the contents of which should not be transformed
+    :param nested_transforms: a mapping of keys to Callable, the Callable will only be passed the value for the key
+                              in the resource dictionary
+    :return: dictionary representing the transformed resource
+    """
+    if not resource:
+        return resource
+    ignore_list = ignore_list or []
+    nested_transforms = nested_transforms or {}
+
+    transformed_resource = deepcopy(resource)
+    if normalize:
+        transformed_resource = normalize_boto3_result(transformed_resource)
+    transformed_resource = _perform_nested_transforms(transformed_resource, nested_transforms)
+    ignore_list = [*ignore_list, *nested_transforms]
+    camel_resource = camel_dict_to_snake_dict(transformed_resource, ignore_list=ignore_list)
+    if transform_tags and "Tags" in resource:
+        camel_resource["tags"] = boto3_tag_list_to_ansible_dict(resource["Tags"])
+    if force_tags and "Tags" not in resource:
+        camel_resource["tags"] = {}
+
+    return camel_resource
+
+
+def boto3_resource_list_to_ansible_dict(
+    resource_list: BotoResourceList,
+    transform_tags: bool = True,
+    force_tags: bool = True,
+    normalize: bool = True,
+    ignore_list: Optional[Sequence[str]] = None,
+    nested_transforms: Optional[Mapping[str, Callable]] = None,
+) -> AnsibleAWSResourceList:
+    """
+    Transforms a list of boto3-style (CamelCase) resources to the ansible-style (snake_case).
+
+    :param resource_list: a list of dictionaries representing the resources
+    :param transform_tags: whether or not to perform "tag list" to "dictionary" conversion on the "Tags" key
+    :param normalize: whether resources should be passed through .botocore.normalize_boto3_result()
+    :param ignore_list: a list of keys, the contents of which should not be transformed
+    :param nested_transforms: a mapping of keys to Callable, the Callable will only be passed the value for the key
+                              in the resource dictionary
+    :return: list of dictionaries representing the transformed resources
+    """
+    if not resource_list:
+        return resource_list
+    return [
+        boto3_resource_to_ansible_dict(resource, transform_tags, force_tags, normalize, ignore_list, nested_transforms)
+        for resource in resource_list
+    ]