From 0f971495570d10c8c91a47517c155c37ed5bc1f5 Mon Sep 17 00:00:00 2001 From: "patchback[bot]" <45432694+patchback[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 22:24:28 +0000 Subject: [PATCH] Normalize refactor (#2006) (#2027) [PR #2006/3974a823 backport][stable-7] Normalize refactor This is a backport of PR #2006 as merged into main (3974a82). SUMMARY fixes: #2001 ISSUE TYPE Feature Pull Request COMPONENT NAME plugins/module_utils/iam.py plugins/module_utils/transformation.py ADDITIONAL INFORMATION Note: also tinkers a little with Type Hinting, not sure of the value TODO: Initial transform code Unit tests for transform code initial iam updates Basic unit tests for iam updates changelog Reviewed-by: Helen Bailey --- .../fragments/2006-normalize-refactor.yml | 3 + plugins/module_utils/iam.py | 191 +++--- plugins/module_utils/transformation.py | 96 +++ .../iam/test_iam_resource_transforms.py | 583 ++++++++++++++++++ .../test_boto3_resource_to_ansible_dict.py | 140 +++++ 5 files changed, 908 insertions(+), 105 deletions(-) create mode 100644 changelogs/fragments/2006-normalize-refactor.yml create mode 100644 tests/unit/module_utils/iam/test_iam_resource_transforms.py create mode 100644 tests/unit/module_utils/transformation/test_boto3_resource_to_ansible_dict.py diff --git a/changelogs/fragments/2006-normalize-refactor.yml b/changelogs/fragments/2006-normalize-refactor.yml new file mode 100644 index 00000000000..db96501ab25 --- /dev/null +++ b/changelogs/fragments/2006-normalize-refactor.yml @@ -0,0 +1,3 @@ +minor_changes: +- module_utils.transformations - add ``boto3_resource_to_ansible_dict()`` and ``boto3_resource_list_to_ansible_dict()`` helpers (https://github.com/ansible-collections/amazon.aws/pull/2006). +- module_utils.iam - refactored normalization functions to use ``boto3_resource_to_ansible_dict()`` and ``boto3_resource_list_to_ansible_dict()`` (https://github.com/ansible-collections/amazon.aws/pull/2006). diff --git a/plugins/module_utils/iam.py b/plugins/module_utils/iam.py index 430823f3b43..56920d53eb3 100644 --- a/plugins/module_utils/iam.py +++ b/plugins/module_utils/iam.py @@ -4,7 +4,6 @@ # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) import re -from copy import deepcopy try: import botocore @@ -12,17 +11,20 @@ pass # Modules are responsible for handling this. from ansible.module_utils._text import to_native -from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict from .arn import parse_aws_arn from .arn import validate_aws_arn from .botocore import is_boto3_error_code -from .botocore import normalize_boto3_result from .errors import AWSErrorHandler from .exceptions import AnsibleAWSError from .retries import AWSRetry from .tagging import ansible_dict_to_boto3_tag_list -from .tagging import boto3_tag_list_to_ansible_dict +from .transformation import AnsibleAWSResource +from .transformation import AnsibleAWSResourceList +from .transformation import BotoResource +from .transformation import BotoResourceList +from .transformation import boto3_resource_list_to_ansible_dict +from .transformation import boto3_resource_to_ansible_dict class AnsibleIAMError(AnsibleAWSError): @@ -198,66 +200,6 @@ def get_iam_managed_policy_version(client, arn, version): return client.get_policy_version(PolicyArn=arn, VersionId=version)["PolicyVersion"] -def normalize_iam_mfa_device(device): - """Converts IAM MFA Device from the CamelCase boto3 format to the snake_case Ansible format""" - if not device: - return device - camel_device = camel_dict_to_snake_dict(device) - camel_device["tags"] = boto3_tag_list_to_ansible_dict(device.pop("Tags", [])) - return camel_device - - -def normalize_iam_mfa_devices(devices): - """Converts a list of IAM MFA Devices from the CamelCase boto3 format to the snake_case Ansible format""" - if not devices: - return [] - devices = [normalize_iam_mfa_device(d) for d in devices] - return devices - - -def normalize_iam_user(user): - """Converts IAM users from the CamelCase boto3 format to the snake_case Ansible format""" - if not user: - return user - camel_user = camel_dict_to_snake_dict(user) - camel_user["tags"] = boto3_tag_list_to_ansible_dict(user.pop("Tags", [])) - return camel_user - - -def normalize_iam_policy(policy): - """Converts IAM policies from the CamelCase boto3 format to the snake_case Ansible format""" - if not policy: - return policy - camel_policy = camel_dict_to_snake_dict(policy) - camel_policy["tags"] = boto3_tag_list_to_ansible_dict(policy.get("Tags", [])) - return camel_policy - - -def normalize_iam_group(group): - """Converts IAM Groups from the CamelCase boto3 format to the snake_case Ansible format""" - if not group: - return group - camel_group = camel_dict_to_snake_dict(normalize_boto3_result(group)) - return camel_group - - -def normalize_iam_access_key(access_key): - """Converts IAM access keys from the CamelCase boto3 format to the snake_case Ansible format""" - if not access_key: - return access_key - camel_key = camel_dict_to_snake_dict(normalize_boto3_result(access_key)) - return camel_key - - -def normalize_iam_access_keys(access_keys): - """Converts a list of IAM access keys from the CamelCase boto3 format to the snake_case Ansible format""" - if not access_keys: - return [] - access_keys = [normalize_iam_access_key(k) for k in access_keys] - sorted_keys = sorted(access_keys, key=lambda d: d.get("create_date", None)) - return sorted_keys - - def convert_managed_policy_names_to_arns(client, policy_names): if all(validate_aws_arn(policy, service="iam") for policy in policy_names if policy is not None): return policy_names @@ -386,47 +328,6 @@ def list_iam_instance_profiles(client, name=None, prefix=None, role=None): return _list_iam_instance_profiles(client) -def normalize_iam_instance_profile(profile, _v7_compat=False): - """ - Converts a boto3 format IAM instance profile into "Ansible" format - - _v7_compat is deprecated and will be removed in release after 2025-05-01 DO NOT USE. - """ - - new_profile = camel_dict_to_snake_dict(deepcopy(profile)) - if profile.get("Roles"): - new_profile["roles"] = [normalize_iam_role(role, _v7_compat=_v7_compat) for role in profile.get("Roles")] - if profile.get("Tags"): - new_profile["tags"] = boto3_tag_list_to_ansible_dict(profile.get("Tags")) - else: - new_profile["tags"] = {} - new_profile["original"] = profile - return new_profile - - -def normalize_iam_role(role, _v7_compat=False): - """ - Converts a boto3 format IAM instance role into "Ansible" format - - _v7_compat is deprecated and will be removed in release after 2025-05-01 DO NOT USE. - """ - - new_role = camel_dict_to_snake_dict(deepcopy(role)) - if role.get("InstanceProfiles"): - new_role["instance_profiles"] = [ - normalize_iam_instance_profile(profile, _v7_compat=_v7_compat) for profile in role.get("InstanceProfiles") - ] - if role.get("AssumeRolePolicyDocument"): - if _v7_compat: - # new_role["assume_role_policy_document"] = role.get("AssumeRolePolicyDocument") - new_role["assume_role_policy_document_raw"] = role.get("AssumeRolePolicyDocument") - else: - new_role["assume_role_policy_document"] = role.get("AssumeRolePolicyDocument") - - new_role["tags"] = boto3_tag_list_to_ansible_dict(role.get("Tags", [])) - return new_role - - @IAMErrorHandler.common_error_handler("tag instance profile") @AWSRetry.jittered_backoff() def tag_iam_instance_profile(client, name, tags): @@ -497,3 +398,83 @@ def validate_iam_identifiers(resource_type, name=None, path=None): return path_problem return None + + +def normalize_iam_mfa_device(device: BotoResource) -> AnsibleAWSResource: + """Converts IAM MFA Device from the CamelCase boto3 format to the snake_case Ansible format""" + # MFA Devices don't support Tags (as of 1.34.52) + return boto3_resource_to_ansible_dict(device) + + +def normalize_iam_mfa_devices(devices: BotoResourceList) -> AnsibleAWSResourceList: + """Converts a list of IAM MFA Devices from the CamelCase boto3 format to the snake_case Ansible format""" + # MFA Devices don't support Tags (as of 1.34.52) + return boto3_resource_list_to_ansible_dict(devices) + + +def normalize_iam_user(user: BotoResource) -> AnsibleAWSResource: + """Converts IAM users from the CamelCase boto3 format to the snake_case Ansible format""" + return boto3_resource_to_ansible_dict(user) + + +def normalize_iam_policy(policy: BotoResource) -> AnsibleAWSResource: + """Converts IAM policies from the CamelCase boto3 format to the snake_case Ansible format""" + return boto3_resource_to_ansible_dict(policy) + + +def normalize_iam_group(group: BotoResource) -> AnsibleAWSResource: + """Converts IAM Groups from the CamelCase boto3 format to the snake_case Ansible format""" + # Groups don't support Tags (as of 1.34.52) + return boto3_resource_to_ansible_dict(group, force_tags=False) + + +def normalize_iam_access_key(access_key: BotoResource) -> AnsibleAWSResource: + """Converts IAM access keys from the CamelCase boto3 format to the snake_case Ansible format""" + # Access Keys don't support Tags (as of 1.34.52) + return boto3_resource_to_ansible_dict(access_key, force_tags=False) + + +def normalize_iam_access_keys(access_keys: BotoResourceList) -> AnsibleAWSResourceList: + """Converts a list of IAM access keys from the CamelCase boto3 format to the snake_case Ansible format""" + # Access Keys don't support Tags (as of 1.34.52) + if not access_keys: + return access_keys + access_keys = boto3_resource_list_to_ansible_dict(access_keys, force_tags=False) + return sorted(access_keys, key=lambda d: d.get("create_date", None)) + + +def normalize_iam_instance_profile(profile: BotoResource) -> AnsibleAWSResource: + """ + Converts a boto3 format IAM instance profile into "Ansible" format + + _v7_compat is deprecated and will be removed in release after 2025-05-01 DO NOT USE. + """ + transforms = {"Roles": _normalize_iam_roles} + transformed_profile = boto3_resource_to_ansible_dict(profile, nested_transforms=transforms) + return transformed_profile + + +def normalize_iam_role(role: BotoResource, _v7_compat: bool = False) -> AnsibleAWSResource: + """ + Converts a boto3 format IAM instance role into "Ansible" format + + _v7_compat is deprecated and will be removed in release after 2025-05-01 DO NOT USE. + """ + transforms = {"InstanceProfiles": _normalize_iam_instance_profiles} + ignore_list = [] if _v7_compat else ["AssumeRolePolicyDocument"] + transformed_role = boto3_resource_to_ansible_dict(role, nested_transforms=transforms, ignore_list=ignore_list) + if _v7_compat and role.get("AssumeRolePolicyDocument"): + transformed_role["assume_role_policy_document_raw"] = role["AssumeRolePolicyDocument"] + return transformed_role + + +def _normalize_iam_instance_profiles(profiles: BotoResourceList) -> AnsibleAWSResourceList: + if not profiles: + return profiles + return [normalize_iam_instance_profile(p) for p in profiles] + + +def _normalize_iam_roles(roles: BotoResourceList) -> AnsibleAWSResourceList: + if not roles: + return roles + return [normalize_iam_role(r) for r in roles] diff --git a/plugins/module_utils/transformation.py b/plugins/module_utils/transformation.py index 708736fc0e3..a5bc23607df 100644 --- a/plugins/module_utils/transformation.py +++ b/plugins/module_utils/transformation.py @@ -28,9 +28,26 @@ # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from copy import deepcopy +from typing import Any +from typing import Callable +from typing import Mapping +from typing import Optional +from typing import Sequence +from typing import Union + +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict from ansible.module_utils.six import integer_types from ansible.module_utils.six import string_types +from .botocore import normalize_boto3_result +from .tagging import boto3_tag_list_to_ansible_dict + +BotoResource = Union[None, Mapping[str, Any]] +BotoResourceList = Union[None, Sequence[Mapping[str, Any]]] +AnsibleAWSResource = Union[None, Mapping[str, Any]] +AnsibleAWSResourceList = Union[None, Sequence[Mapping[str, Any]]] + def ansible_dict_to_boto3_filter_list(filters_dict): """Convert an Ansible dict of filters to list of dicts that boto3 can use @@ -133,3 +150,82 @@ def scrub_none_parameters(parameters, descend_into_lists=True): clean_parameters[k] = v return clean_parameters + + +def _perform_nested_transforms( + resource: Mapping[str, Any], + nested_transforms: Optional[Mapping[str, Callable]], +) -> Mapping[str, Any]: + if not nested_transforms: + return resource + + for k, transform in nested_transforms.items(): + if k in resource: + resource[k] = transform(resource[k]) + + return resource + + +def boto3_resource_to_ansible_dict( + resource: BotoResource, + transform_tags: bool = True, + force_tags: bool = True, + normalize: bool = True, + ignore_list: Optional[Sequence[str]] = None, + nested_transforms: Optional[Mapping[str, Callable]] = None, +) -> AnsibleAWSResource: + """ + Transforms boto3-style (CamelCase) resource to the ansible-style (snake_case). + + :param resource: a dictionary representing the resource + :param transform_tags: whether or not to perform "tag list" to "dictionary" conversion on the "Tags" key + :param normalize: whether resources should be passed through .botocore.normalize_boto3_result + :param ignore_list: a list of keys, the contents of which should not be transformed + :param nested_transforms: a mapping of keys to Callable, the Callable will only be passed the value for the key + in the resource dictionary + :return: dictionary representing the transformed resource + """ + if not resource: + return resource + ignore_list = ignore_list or [] + nested_transforms = nested_transforms or {} + + transformed_resource = deepcopy(resource) + if normalize: + transformed_resource = normalize_boto3_result(transformed_resource) + transformed_resource = _perform_nested_transforms(transformed_resource, nested_transforms) + ignore_list = [*ignore_list, *nested_transforms] + camel_resource = camel_dict_to_snake_dict(transformed_resource, ignore_list=ignore_list) + if transform_tags and "Tags" in resource: + camel_resource["tags"] = boto3_tag_list_to_ansible_dict(resource["Tags"]) + if force_tags and "Tags" not in resource: + camel_resource["tags"] = {} + + return camel_resource + + +def boto3_resource_list_to_ansible_dict( + resource_list: BotoResourceList, + transform_tags: bool = True, + force_tags: bool = True, + normalize: bool = True, + ignore_list: Optional[Sequence[str]] = None, + nested_transforms: Optional[Mapping[str, Callable]] = None, +) -> AnsibleAWSResourceList: + """ + Transforms a list of boto3-style (CamelCase) resources to the ansible-style (snake_case). + + :param resource_list: a list of dictionaries representing the resources + :param transform_tags: whether or not to perform "tag list" to "dictionary" conversion on the "Tags" key + :param normalize: whether resources should be passed through .botocore.normalize_boto3_result() + :param ignore_list: a list of keys, the contents of which should not be transformed + :param nested_transforms: a mapping of keys to Callable, the Callable will only be passed the value for the key + in the resource dictionary + :return: list of dictionaries representing the transformed resources + """ + if not resource_list: + return resource_list + return [ + boto3_resource_to_ansible_dict(resource, transform_tags, force_tags, normalize, ignore_list, nested_transforms) + for resource in resource_list + ] diff --git a/tests/unit/module_utils/iam/test_iam_resource_transforms.py b/tests/unit/module_utils/iam/test_iam_resource_transforms.py new file mode 100644 index 00000000000..28090f993ae --- /dev/null +++ b/tests/unit/module_utils/iam/test_iam_resource_transforms.py @@ -0,0 +1,583 @@ +# -*- coding: utf-8 -*- + +# Copyright: Contributors to the Ansible project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + + +import dateutil + +from ansible_collections.amazon.aws.plugins.module_utils.iam import normalize_iam_access_key +from ansible_collections.amazon.aws.plugins.module_utils.iam import normalize_iam_access_keys +from ansible_collections.amazon.aws.plugins.module_utils.iam import normalize_iam_group +from ansible_collections.amazon.aws.plugins.module_utils.iam import normalize_iam_instance_profile +from ansible_collections.amazon.aws.plugins.module_utils.iam import normalize_iam_mfa_device +from ansible_collections.amazon.aws.plugins.module_utils.iam import normalize_iam_mfa_devices +from ansible_collections.amazon.aws.plugins.module_utils.iam import normalize_iam_policy +from ansible_collections.amazon.aws.plugins.module_utils.iam import normalize_iam_role +from ansible_collections.amazon.aws.plugins.module_utils.iam import normalize_iam_user + +# The various normalize_ functions are based upon ..transformation.boto3_resource_to_ansible_dict +# As such these tests will be relatively light touch. + +example_date1_txt = "2020-12-30T00:00:00.000Z" +example_date2_txt = "2021-04-26T01:23:58.000Z" +example_date1_iso = "2020-12-30T00:00:00+00:00" +example_date2_iso = "2021-04-26T01:23:58+00:00" +example_date1 = dateutil.parser.parse(example_date1_txt) +example_date2 = dateutil.parser.parse(example_date2_txt) + + +class TestIamResourceToAnsibleDict: + def setup_method(self): + pass + + def test_normalize_iam_mfa_device(self): + INPUT = { + "UserName": "ExampleUser", + "SerialNumber": "arn:aws:iam::123456789012:mfa/ExampleUser", + "EnableDate": example_date1, + } + OUTPUT = { + "user_name": "ExampleUser", + "serial_number": "arn:aws:iam::123456789012:mfa/ExampleUser", + "enable_date": example_date1_iso, + "tags": {}, + } + + assert OUTPUT == normalize_iam_mfa_device(INPUT) + + def test_normalize_iam_mfa_devices(self): + INPUT = [ + { + "UserName": "ExampleUser", + "SerialNumber": "arn:aws:iam::123456789012:mfa/ExampleUser", + "EnableDate": example_date1, + } + ] + OUTPUT = [ + { + "user_name": "ExampleUser", + "serial_number": "arn:aws:iam::123456789012:mfa/ExampleUser", + "enable_date": example_date1_iso, + "tags": {}, + } + ] + + assert OUTPUT == normalize_iam_mfa_devices(INPUT) + + def test_normalize_iam_user(self): + INPUT = { + "Path": "/MyPath/", + "UserName": "ExampleUser", + "UserId": "AIDU12345EXAMPLE12345", + "Arn": "arn:aws:iam::123456789012:user/MyPath/ExampleUser", + "CreateDate": example_date1, + "PasswordLastUsed": example_date2, + "PermissionsBoundary": { + "PermissionsBoundaryType": "PermissionsBoundaryPolicy", + "PermissionsBoundaryArn": "arn:aws:iam::123456789012:policy/ExamplePolicy", + }, + "Tags": [ + {"Key": "MyKey", "Value": "Example Value"}, + ], + } + + OUTPUT = { + "path": "/MyPath/", + "user_name": "ExampleUser", + "user_id": "AIDU12345EXAMPLE12345", + "arn": "arn:aws:iam::123456789012:user/MyPath/ExampleUser", + "create_date": example_date1_iso, + "password_last_used": example_date2_iso, + "permissions_boundary": { + "permissions_boundary_type": "PermissionsBoundaryPolicy", + "permissions_boundary_arn": "arn:aws:iam::123456789012:policy/ExamplePolicy", + }, + "tags": {"MyKey": "Example Value"}, + } + + assert OUTPUT == normalize_iam_user(INPUT) + + def test_normalize_iam_policy(self): + INPUT = { + "PolicyName": "AnsibleIntegratation-CI-ApplicationServices", + "PolicyId": "ANPA12345EXAMPLE12345", + "Arn": "arn:aws:iam::123456789012:policy/AnsibleIntegratation-CI-ApplicationServices", + "Path": "/examples/", + "DefaultVersionId": "v6", + "AttachmentCount": 2, + "PermissionsBoundaryUsageCount": 0, + "IsAttachable": True, + "CreateDate": example_date1, + "UpdateDate": example_date2, + "Tags": [ + {"Key": "MyKey", "Value": "Example Value"}, + ], + } + + OUTPUT = { + "policy_name": "AnsibleIntegratation-CI-ApplicationServices", + "policy_id": "ANPA12345EXAMPLE12345", + "arn": "arn:aws:iam::123456789012:policy/AnsibleIntegratation-CI-ApplicationServices", + "path": "/examples/", + "default_version_id": "v6", + "attachment_count": 2, + "permissions_boundary_usage_count": 0, + "is_attachable": True, + "create_date": example_date1_iso, + "update_date": example_date2_iso, + "tags": {"MyKey": "Example Value"}, + } + + assert OUTPUT == normalize_iam_policy(INPUT) + + def test_normalize_iam_group(self): + INPUT = { + "Users": [ + { + "Path": "/", + "UserName": "ansible_test", + "UserId": "AIDA12345EXAMPLE12345", + "Arn": "arn:aws:iam::123456789012:user/ansible_test", + "CreateDate": example_date1, + "PasswordLastUsed": example_date2, + } + ], + "Group": { + "Path": "/", + "GroupName": "ansible-integration-ci", + "GroupId": "AGPA01234EXAMPLE01234", + "Arn": "arn:aws:iam::123456789012:group/ansible-integration-ci", + "CreateDate": example_date1, + }, + "AttachedPolicies": [ + { + "PolicyName": "AnsibleIntegratation-CI-Paas", + "PolicyArn": "arn:aws:iam::123456789012:policy/AnsibleIntegratation-CI-Paas", + }, + { + "PolicyName": "AnsibleIntegratation-CI-ApplicationServices", + "PolicyArn": "arn:aws:iam::123456789012:policy/AnsibleIntegratation-CI-ApplicationServices", + }, + ], + } + + OUTPUT = { + "users": [ + { + "path": "/", + "user_name": "ansible_test", + "user_id": "AIDA12345EXAMPLE12345", + "arn": "arn:aws:iam::123456789012:user/ansible_test", + "create_date": example_date1_iso, + "password_last_used": example_date2_iso, + } + ], + "group": { + "path": "/", + "group_name": "ansible-integration-ci", + "group_id": "AGPA01234EXAMPLE01234", + "arn": "arn:aws:iam::123456789012:group/ansible-integration-ci", + "create_date": example_date1_iso, + }, + "attached_policies": [ + { + "policy_name": "AnsibleIntegratation-CI-Paas", + "policy_arn": "arn:aws:iam::123456789012:policy/AnsibleIntegratation-CI-Paas", + }, + { + "policy_name": "AnsibleIntegratation-CI-ApplicationServices", + "policy_arn": "arn:aws:iam::123456789012:policy/AnsibleIntegratation-CI-ApplicationServices", + }, + ], + } + + assert OUTPUT == normalize_iam_group(INPUT) + + def test_normalize_access_key(self): + INPUT = { + "UserName": "ansible_test", + "AccessKeyId": "AKIA12345EXAMPLE1234", + "Status": "Active", + "CreateDate": example_date1, + } + + OUTPUT = { + "user_name": "ansible_test", + "access_key_id": "AKIA12345EXAMPLE1234", + "status": "Active", + "create_date": example_date1_iso, + } + + assert OUTPUT == normalize_iam_access_key(INPUT) + + def test_normalize_access_keys(self): + INPUT = [ + { + "UserName": "ansible_test", + "AccessKeyId": "AKIA12345EXAMPLE1234", + "Status": "Active", + "CreateDate": example_date1, + }, + { + "UserName": "ansible_test", + "AccessKeyId": "AKIA01234EXAMPLE4321", + "Status": "Active", + "CreateDate": example_date2, + }, + ] + + OUTPUT = [ + { + "access_key_id": "AKIA12345EXAMPLE1234", + "create_date": example_date1_iso, + "status": "Active", + "user_name": "ansible_test", + }, + { + "access_key_id": "AKIA01234EXAMPLE4321", + "create_date": example_date2_iso, + "status": "Active", + "user_name": "ansible_test", + }, + ] + + assert OUTPUT == normalize_iam_access_keys(INPUT) + + # Switch order to test that they're sorted by Creation Date + INPUT = [ + { + "UserName": "ansible_test", + "AccessKeyId": "AKIA12345EXAMPLE1234", + "Status": "Active", + "CreateDate": example_date2, + }, + { + "UserName": "ansible_test", + "AccessKeyId": "AKIA01234EXAMPLE4321", + "Status": "Active", + "CreateDate": example_date1, + }, + ] + + OUTPUT = [ + { + "access_key_id": "AKIA01234EXAMPLE4321", + "create_date": example_date1_iso, + "status": "Active", + "user_name": "ansible_test", + }, + { + "access_key_id": "AKIA12345EXAMPLE1234", + "create_date": example_date2_iso, + "status": "Active", + "user_name": "ansible_test", + }, + ] + + assert OUTPUT == normalize_iam_access_keys(INPUT) + + def test_normalize_role(self): + INPUT = { + "Arn": "arn:aws:iam::123456789012:role/ansible-test-76640355", + "AssumeRolePolicyDocument": { + "Statement": [ + {"Action": "sts:AssumeRole", "Effect": "Deny", "Principal": {"Service": "ec2.amazonaws.com"}} + ], + "Version": "2012-10-17", + }, + "CreateDate": example_date1, + "Description": "Ansible Test Role (updated) ansible-test-76640355", + "InlinePolicies": ["inline-policy-a", "inline-policy-b"], + "InstanceProfiles": [ + { + "Arn": "arn:aws:iam::123456789012:instance-profile/ansible-test-76640355", + "CreateDate": example_date2, + "InstanceProfileId": "AIPA12345EXAMPLE12345", + "InstanceProfileName": "ansible-test-76640355", + "Path": "/", + "Roles": [ + { + "Arn": "arn:aws:iam::123456789012:role/ansible-test-76640355", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Deny", + "Principal": {"Service": "ec2.amazonaws.com"}, + } + ], + "Version": "2012-10-17", + }, + "CreateDate": example_date1, + "Path": "/", + "RoleId": "AROA12345EXAMPLE12345", + "RoleName": "ansible-test-76640355", + # XXX Bug in iam_role_info - Tags should have been in here. + "Tags": [{"Key": "TagB", "Value": "ValueB"}], + } + ], + "Tags": [{"Key": "TagA", "Value": "Value A"}], + } + ], + "ManagedPolicies": [ + { + "PolicyArn": "arn:aws:iam::123456789012:policy/ansible-test-76640355", + "PolicyName": "ansible-test-76640355", + } + ], + "MaxSessionDuration": 43200, + "Path": "/", + "RoleId": "AROA12345EXAMPLE12345", + "RoleLastUsed": {}, + "RoleName": "ansible-test-76640355", + "Tags": [{"Key": "TagB", "Value": "ValueB"}], + } + + OUTPUT = { + "arn": "arn:aws:iam::123456789012:role/ansible-test-76640355", + "assume_role_policy_document": { + "Statement": [ + {"Action": "sts:AssumeRole", "Effect": "Deny", "Principal": {"Service": "ec2.amazonaws.com"}} + ], + "Version": "2012-10-17", + }, + "create_date": example_date1_iso, + "description": "Ansible Test Role (updated) ansible-test-76640355", + "inline_policies": ["inline-policy-a", "inline-policy-b"], + "instance_profiles": [ + { + "arn": "arn:aws:iam::123456789012:instance-profile/ansible-test-76640355", + "create_date": example_date2_iso, + "instance_profile_id": "AIPA12345EXAMPLE12345", + "instance_profile_name": "ansible-test-76640355", + "path": "/", + "roles": [ + { + "arn": "arn:aws:iam::123456789012:role/ansible-test-76640355", + "assume_role_policy_document": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Deny", + "Principal": {"Service": "ec2.amazonaws.com"}, + } + ], + "Version": "2012-10-17", + }, + "create_date": example_date1_iso, + "path": "/", + "role_id": "AROA12345EXAMPLE12345", + "role_name": "ansible-test-76640355", + "tags": {"TagB": "ValueB"}, + } + ], + "tags": {"TagA": "Value A"}, + } + ], + "managed_policies": [ + { + "policy_arn": "arn:aws:iam::123456789012:policy/ansible-test-76640355", + "policy_name": "ansible-test-76640355", + } + ], + "max_session_duration": 43200, + "path": "/", + "role_id": "AROA12345EXAMPLE12345", + "role_last_used": {}, + "role_name": "ansible-test-76640355", + "tags": {"TagB": "ValueB"}, + } + + assert OUTPUT == normalize_iam_role(INPUT) + + def test_normalize_role_compat(self): + INPUT = { + "Arn": "arn:aws:iam::123456789012:role/ansible-test-76640355", + "AssumeRolePolicyDocument": { + "Statement": [ + {"Action": "sts:AssumeRole", "Effect": "Deny", "Principal": {"Service": "ec2.amazonaws.com"}} + ], + "Version": "2012-10-17", + }, + "CreateDate": example_date1, + "Description": "Ansible Test Role (updated) ansible-test-76640355", + "InlinePolicies": ["inline-policy-a", "inline-policy-b"], + "InstanceProfiles": [ + { + "Arn": "arn:aws:iam::123456789012:instance-profile/ansible-test-76640355", + "CreateDate": example_date2, + "InstanceProfileId": "AIPA12345EXAMPLE12345", + "InstanceProfileName": "ansible-test-76640355", + "Path": "/", + "Roles": [ + { + "Arn": "arn:aws:iam::123456789012:role/ansible-test-76640355", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Deny", + "Principal": {"Service": "ec2.amazonaws.com"}, + } + ], + "Version": "2012-10-17", + }, + "CreateDate": example_date1, + "Path": "/", + "RoleId": "AROA12345EXAMPLE12345", + "RoleName": "ansible-test-76640355", + # XXX Bug in iam_role_info - Tags should have been in here. + "Tags": [{"Key": "TagB", "Value": "ValueB"}], + } + ], + "Tags": [{"Key": "TagA", "Value": "Value A"}], + } + ], + "ManagedPolicies": [ + { + "PolicyArn": "arn:aws:iam::123456789012:policy/ansible-test-76640355", + "PolicyName": "ansible-test-76640355", + } + ], + "MaxSessionDuration": 43200, + "Path": "/", + "RoleId": "AROA12345EXAMPLE12345", + "RoleLastUsed": {}, + "RoleName": "ansible-test-76640355", + "Tags": [{"Key": "TagB", "Value": "ValueB"}], + } + + OUTPUT = { + "arn": "arn:aws:iam::123456789012:role/ansible-test-76640355", + "assume_role_policy_document": { + "statement": [ + {"action": "sts:AssumeRole", "effect": "Deny", "principal": {"service": "ec2.amazonaws.com"}} + ], + "version": "2012-10-17", + }, + "assume_role_policy_document_raw": { + "Statement": [ + {"Action": "sts:AssumeRole", "Effect": "Deny", "Principal": {"Service": "ec2.amazonaws.com"}} + ], + "Version": "2012-10-17", + }, + "create_date": example_date1_iso, + "description": "Ansible Test Role (updated) ansible-test-76640355", + "inline_policies": ["inline-policy-a", "inline-policy-b"], + "instance_profiles": [ + { + "arn": "arn:aws:iam::123456789012:instance-profile/ansible-test-76640355", + "create_date": example_date2_iso, + "instance_profile_id": "AIPA12345EXAMPLE12345", + "instance_profile_name": "ansible-test-76640355", + "path": "/", + "roles": [ + { + "arn": "arn:aws:iam::123456789012:role/ansible-test-76640355", + "assume_role_policy_document": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Deny", + "Principal": {"Service": "ec2.amazonaws.com"}, + } + ], + "Version": "2012-10-17", + }, + "create_date": example_date1_iso, + "path": "/", + "role_id": "AROA12345EXAMPLE12345", + "role_name": "ansible-test-76640355", + "tags": {"TagB": "ValueB"}, + } + ], + "tags": {"TagA": "Value A"}, + } + ], + "managed_policies": [ + { + "policy_arn": "arn:aws:iam::123456789012:policy/ansible-test-76640355", + "policy_name": "ansible-test-76640355", + } + ], + "max_session_duration": 43200, + "path": "/", + "role_id": "AROA12345EXAMPLE12345", + "role_last_used": {}, + "role_name": "ansible-test-76640355", + "tags": {"TagB": "ValueB"}, + } + + assert OUTPUT == normalize_iam_role(INPUT, _v7_compat=True) + + def test_normalize_instance_profile(self): + INPUT = { + "Arn": "arn:aws:iam::123456789012:instance-profile/ansible-test-40050922/ansible-test-40050922", + "CreateDate": example_date1, + "InstanceProfileId": "AIPA12345EXAMPLE12345", + "InstanceProfileName": "ansible-test-40050922", + "Path": "/ansible-test-40050922/", + "Roles": [ + { + "Arn": "arn:aws:iam::123456789012:role/ansible-test-40050922/ansible-test-40050922", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Deny", + "Principal": {"Service": "ec2.amazonaws.com"}, + } + ], + "Version": "2012-10-17", + }, + "CreateDate": example_date2, + "Path": "/ansible-test-40050922/", + "RoleId": "AROA12345EXAMPLE12345", + "RoleName": "ansible-test-40050922", + "Tags": [{"Key": "TagC", "Value": "ValueC"}], + } + ], + "Tags": [ + {"Key": "Key with Spaces", "Value": "Value with spaces"}, + {"Key": "snake_case_key", "Value": "snake_case_value"}, + {"Key": "CamelCaseKey", "Value": "CamelCaseValue"}, + {"Key": "pascalCaseKey", "Value": "pascalCaseValue"}, + ], + } + + OUTPUT = { + "arn": "arn:aws:iam::123456789012:instance-profile/ansible-test-40050922/ansible-test-40050922", + "create_date": "2020-12-30T00:00:00+00:00", + "instance_profile_id": "AIPA12345EXAMPLE12345", + "instance_profile_name": "ansible-test-40050922", + "path": "/ansible-test-40050922/", + "roles": [ + { + "arn": "arn:aws:iam::123456789012:role/ansible-test-40050922/ansible-test-40050922", + "assume_role_policy_document": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Deny", + "Principal": {"Service": "ec2.amazonaws.com"}, + } + ], + "Version": "2012-10-17", + }, + "create_date": "2021-04-26T01:23:58+00:00", + "path": "/ansible-test-40050922/", + "role_id": "AROA12345EXAMPLE12345", + "role_name": "ansible-test-40050922", + "tags": {"TagC": "ValueC"}, + } + ], + "tags": { + "CamelCaseKey": "CamelCaseValue", + "Key with Spaces": "Value with spaces", + "pascalCaseKey": "pascalCaseValue", + "snake_case_key": "snake_case_value", + }, + } + + assert OUTPUT == normalize_iam_instance_profile(INPUT) diff --git a/tests/unit/module_utils/transformation/test_boto3_resource_to_ansible_dict.py b/tests/unit/module_utils/transformation/test_boto3_resource_to_ansible_dict.py new file mode 100644 index 00000000000..89a0a837cab --- /dev/null +++ b/tests/unit/module_utils/transformation/test_boto3_resource_to_ansible_dict.py @@ -0,0 +1,140 @@ +# (c) 2017 Red Hat Inc. +# +# This file is part of Ansible +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from copy import deepcopy +from unittest.mock import sentinel + +import dateutil +import pytest + +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + +from ansible_collections.amazon.aws.plugins.module_utils.transformation import boto3_resource_to_ansible_dict + +example_date_txt = "2020-12-30T00:00:00.000Z" +example_date_iso = "2020-12-30T00:00:00+00:00" +example_date = dateutil.parser.parse(example_date_txt) + +EXAMPLE_BOTO3 = [ + None, + {}, + {"ExampleDate": example_date}, + {"ExampleTxtDate": example_date_txt}, + {"Tags": [{"Key": "MyKey", "Value": "MyValue"}, {"Key": "Normal case", "Value": "Normal Value"}]}, + { + "Name": "ExampleResource", + "ExampleDate": example_date, + "Tags": [{"Key": "MyKey", "Value": "MyValue"}, {"Key": "Normal case", "Value": "Normal Value"}], + }, + {"ExampleNested": {"ExampleKey": "Example Value"}}, +] + +EXAMPLE_DICT = [ + None, + {}, + {"example_date": example_date_iso, "tags": {}}, + {"example_txt_date": example_date_txt, "tags": {}}, + {"tags": {"MyKey": "MyValue", "Normal case": "Normal Value"}}, + { + "name": "ExampleResource", + "example_date": example_date_iso, + "tags": {"MyKey": "MyValue", "Normal case": "Normal Value"}, + }, + {"example_nested": {"example_key": "Example Value"}, "tags": {}}, +] + +TEST_DATA = zip(EXAMPLE_BOTO3, EXAMPLE_DICT) + +NESTED_DATA = {"sentinal": sentinel.MY_VALUE} + + +def do_transform_nested(resource): + return {"sentinal": sentinel.MY_VALUE} + + +class TestBoto3ResourceToAnsibleDict: + def setup_method(self): + pass + + @pytest.mark.parametrize("input_params, output_params", deepcopy(TEST_DATA)) + def test_default_conversion(self, input_params, output_params): + # Test default behaviour + assert boto3_resource_to_ansible_dict(input_params) == output_params + + @pytest.mark.parametrize("input_params, output_params", deepcopy(TEST_DATA)) + def test_normalize(self, input_params, output_params): + # Test with normalize explicitly enabled + assert boto3_resource_to_ansible_dict(input_params, normalize=True) == output_params + + @pytest.mark.parametrize("input_params, output_params", deepcopy(TEST_DATA)) + def test_no_normalize(self, input_params, output_params): + # Test with normalize explicitly disabled + expected_value = deepcopy(output_params) + if input_params and "ExampleDate" in input_params: + expected_value["example_date"] = example_date + assert expected_value == boto3_resource_to_ansible_dict(input_params, normalize=False) + + @pytest.mark.parametrize("input_params, output_params", deepcopy(TEST_DATA)) + def test_no_skip(self, input_params, output_params): + # Test with ignore_list explicitly set to [] + assert boto3_resource_to_ansible_dict(input_params, ignore_list=[]) == output_params + assert boto3_resource_to_ansible_dict(input_params, ignore_list=["NotUsed"]) == output_params + + @pytest.mark.parametrize("input_params, output_params", deepcopy(TEST_DATA)) + def test_skip(self, input_params, output_params): + # Test with ignore_list explicitly set + expected_value = deepcopy(output_params) + if input_params and "ExampleNested" in input_params: + expected_value["example_nested"] = input_params["ExampleNested"] + assert expected_value == boto3_resource_to_ansible_dict(input_params, ignore_list=["ExampleNested"]) + assert expected_value == boto3_resource_to_ansible_dict(input_params, ignore_list=["NotUsed", "ExampleNested"]) + assert expected_value == boto3_resource_to_ansible_dict(input_params, ignore_list=["ExampleNested", "NotUsed"]) + + @pytest.mark.parametrize("input_params, output_params", deepcopy(TEST_DATA)) + def test_tags(self, input_params, output_params): + # Test with transform_tags explicitly enabled + assert boto3_resource_to_ansible_dict(input_params, transform_tags=True) == output_params + + @pytest.mark.parametrize("input_params, output_params", deepcopy(TEST_DATA)) + def test_no_tags(self, input_params, output_params): + # Test with transform_tags explicitly disabled + expected_value = deepcopy(output_params) + if input_params and "Tags" in input_params: + camel_tags = camel_dict_to_snake_dict({"tags": input_params["Tags"]}) + expected_value.update(camel_tags) + assert expected_value == boto3_resource_to_ansible_dict(input_params, transform_tags=False) + + @pytest.mark.parametrize("input_params, output_params", deepcopy(TEST_DATA)) + def test_no_nested(self, input_params, output_params): + # Test with transform_nested explicitly set to an empty dictionary + assert boto3_resource_to_ansible_dict(input_params, nested_transforms={}) == output_params + + @pytest.mark.parametrize("input_params, output_params", deepcopy(TEST_DATA)) + def test_nested(self, input_params, output_params): + # Test with a custom transformation of nested resources + transform_map = {"ExampleNested": do_transform_nested} + expected_value = deepcopy(output_params) + + actual_value = boto3_resource_to_ansible_dict(input_params, nested_transforms=transform_map) + + if input_params and "ExampleNested" in input_params: + assert actual_value["example_nested"] == NESTED_DATA + del actual_value["example_nested"] + del expected_value["example_nested"] + + assert expected_value == actual_value + + @pytest.mark.parametrize("input_params, output_params", deepcopy(TEST_DATA)) + def test_force_tags(self, input_params, output_params): + # Test with force_tags explicitly enabled + assert boto3_resource_to_ansible_dict(input_params, force_tags=True) == output_params + + @pytest.mark.parametrize("input_params, output_params", deepcopy(TEST_DATA)) + def test_no_force_tags(self, input_params, output_params): + # Test with force_tags explicitly enabled + expected_value = deepcopy(output_params) + if input_params and "Tags" not in input_params: + del expected_value["tags"] + assert boto3_resource_to_ansible_dict(input_params, force_tags=False) == expected_value