diff --git a/changelogs/fragments/532-rds_param_group-fix.yml b/changelogs/fragments/532-rds_param_group-fix.yml new file mode 100644 index 00000000000..a99c73425f6 --- /dev/null +++ b/changelogs/fragments/532-rds_param_group-fix.yml @@ -0,0 +1,4 @@ +minor_changes: +- rds_param_group - Add AWSRetry (https://github.com/ansible-collections/community.aws/pull/532). +- rds_param_group - Fix integration tests (https://github.com/ansible-collections/community.aws/pull/532). +- rds_param_group - Support check_mode (https://github.com/ansible-collections/community.aws/pull/532). \ No newline at end of file diff --git a/plugins/modules/rds_param_group.py b/plugins/modules/rds_param_group.py index 30aa814de67..ab0718e4b04 100644 --- a/plugins/modules/rds_param_group.py +++ b/plugins/modules/rds_param_group.py @@ -120,11 +120,11 @@ from ansible.module_utils.parsing.convert_bool import BOOLEANS_TRUE from ansible.module_utils.six import string_types -from ansible.module_utils._text import to_native from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_tag_list from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags @@ -137,6 +137,15 @@ } +@AWSRetry.jittered_backoff() +def _describe_db_parameters(connection, **params): + try: + paginator = connection.get_paginator('describe_db_parameters') + return paginator.paginate(**params).build_full_result() + except is_boto3_error_code('DBParameterGroupNotFound'): + return None + + def convert_parameter(param, value): """ Allows setting parameters with 10M = 10* 1024 * 1024 and so on. @@ -158,7 +167,7 @@ def convert_parameter(param, value): elif param['DataType'] == 'boolean': if isinstance(value, string_types): - converted_value = to_native(value) in BOOLEANS_TRUE + converted_value = value in BOOLEANS_TRUE # convert True/False to 1/0 converted_value = 1 if converted_value else 0 return str(converted_value) @@ -170,8 +179,13 @@ def update_parameters(module, connection): apply_method = 'immediate' if module.params['immediate'] else 'pending-reboot' errors = [] modify_list = [] - parameters_paginator = connection.get_paginator('describe_db_parameters') - existing = parameters_paginator.paginate(DBParameterGroupName=groupname).build_full_result()['Parameters'] + existing = {} + try: + _existing = _describe_db_parameters(connection, DBParameterGroupName=groupname) + if _existing: + existing = _existing['Parameters'] + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Failed to describe existing parameter groups") lookup = dict((param['ParameterName'], param) for param in existing) for param_key, param_value in desired.items(): if param_key not in lookup: @@ -187,7 +201,7 @@ def update_parameters(module, connection): errors.append("Parameter %s is not modifiable" % param_key) # modify_db_parameters takes at most 20 parameters - if modify_list: + if modify_list and not module.check_mode: try: from itertools import izip_longest as zip_longest # python 2 except ImportError: @@ -195,7 +209,7 @@ def update_parameters(module, connection): for modify_slice in zip_longest(*[iter(modify_list)] * 20, fillvalue=None): non_empty_slice = [item for item in modify_slice if item] try: - connection.modify_db_parameter_group(DBParameterGroupName=groupname, Parameters=non_empty_slice) + connection.modify_db_parameter_group(aws_retry=True, DBParameterGroupName=groupname, Parameters=non_empty_slice) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg="Couldn't update parameters") return True, errors @@ -204,19 +218,26 @@ def update_parameters(module, connection): def update_tags(module, connection, group, tags): changed = False - existing_tags = connection.list_tags_for_resource(ResourceName=group['DBParameterGroupArn'])['TagList'] + existing_tags = connection.list_tags_for_resource(aws_retry=True, ResourceName=group['DBParameterGroupArn'])['TagList'] to_update, to_delete = compare_aws_tags(boto3_tag_list_to_ansible_dict(existing_tags), tags, module.params['purge_tags']) + + if module.check_mode: + if not to_update and not to_delete: + return False + else: + return True + if to_update: try: - connection.add_tags_to_resource(ResourceName=group['DBParameterGroupArn'], + connection.add_tags_to_resource(aws_retry=True, ResourceName=group['DBParameterGroupArn'], Tags=ansible_dict_to_boto3_tag_list(to_update)) changed = True except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg="Couldn't add tags to parameter group") if to_delete: try: - connection.remove_tags_from_resource(ResourceName=group['DBParameterGroupArn'], + connection.remove_tags_from_resource(aws_retry=True, ResourceName=group['DBParameterGroupArn'], TagKeys=to_delete) changed = True except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: @@ -230,7 +251,7 @@ def ensure_present(module, connection): changed = False errors = [] try: - response = connection.describe_db_parameter_groups(DBParameterGroupName=groupname) + response = connection.describe_db_parameter_groups(aws_retry=True, DBParameterGroupName=groupname) except is_boto3_error_code('DBParameterGroupNotFound'): response = None except botocore.exceptions.ClientError as e: # pylint: disable=duplicate-except @@ -241,11 +262,12 @@ def ensure_present(module, connection): Description=module.params['description']) if tags: params['Tags'] = ansible_dict_to_boto3_tag_list(tags) - try: - response = connection.create_db_parameter_group(**params) - changed = True - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - module.fail_json_aws(e, msg="Couldn't create parameter group") + if not module.check_mode: + try: + response = connection.create_db_parameter_group(aws_retry=True, **params) + changed = True + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Couldn't create parameter group") else: group = response['DBParameterGroups'][0] if tags: @@ -256,12 +278,14 @@ def ensure_present(module, connection): changed = changed or params_changed try: - response = connection.describe_db_parameter_groups(DBParameterGroupName=groupname) + response = connection.describe_db_parameter_groups(aws_retry=True, DBParameterGroupName=groupname) group = camel_dict_to_snake_dict(response['DBParameterGroups'][0]) - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + except is_boto3_error_code('DBParameterGroupNotFound'): + module.exit_json(changed=True, errors=errors) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Couldn't obtain parameter group information") try: - tags = connection.list_tags_for_resource(ResourceName=group['db_parameter_group_arn'])['TagList'] + tags = connection.list_tags_for_resource(aws_retry=True, ResourceName=group['db_parameter_group_arn'])['TagList'] except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg="Couldn't obtain parameter group tags") group['tags'] = boto3_tag_list_to_ansible_dict(tags) @@ -277,8 +301,12 @@ def ensure_absent(module, connection): module.exit_json(changed=False) except botocore.exceptions.ClientError as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Couldn't access parameter group information") + + if response and module.check_mode: + module.exit_json(changed=True) + try: - response = connection.delete_db_parameter_group(DBParameterGroupName=group) + response = connection.delete_db_parameter_group(aws_retry=True, DBParameterGroupName=group) module.exit_json(changed=True) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg="Couldn't delete parameter group") @@ -298,10 +326,11 @@ def main(): module = AnsibleAWSModule( argument_spec=argument_spec, required_if=[['state', 'present', ['description', 'engine']]], + supports_check_mode=True ) try: - conn = module.client('rds') + conn = module.client('rds', retry_decorator=AWSRetry.jittered_backoff()) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg='Failed to connect to AWS') diff --git a/tests/integration/targets/rds_param_group/tasks/main.yml b/tests/integration/targets/rds_param_group/tasks/main.yml index 9af2776b3e0..677260d5ee1 100644 --- a/tests/integration/targets/rds_param_group/tasks/main.yml +++ b/tests/integration/targets/rds_param_group/tasks/main.yml @@ -13,42 +13,73 @@ # ============================================================ # - include: ../../setup_ec2/tasks/common.yml module_name=rds_param_group -- block: +- name: rds_option_group tests + collections: + - amazon.aws + + module_defaults: + group/aws: + ec2_access_key: '{{ aws_access_key }}' + ec2_secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token | default(omit) }}' + region: '{{ ec2_region }}' + + block: # ============================================================ + - name: test empty parameter group - CHECK_MODE + rds_param_group: + name: "{{ rds_param_group.name }}" + engine: "{{ rds_param_group.engine }}" + description: "{{ rds_param_group.description }}" + state: present + check_mode: true + register: result + + - name: assert rds parameter group changed - CHECK_MODE + assert: + that: + - 'result.changed' + - name: test empty parameter group rds_param_group: name: "{{ rds_param_group.name }}" engine: "{{ rds_param_group.engine }}" description: "{{ rds_param_group.description }}" state: present - region: "{{ ec2_region }}" - ec2_access_key: '{{ aws_access_key }}' - ec2_secret_key: '{{ aws_secret_key }}' - security_token: '{{ security_token }}' register: result - ignore_errors: true - name: assert rds parameter group changed assert: that: - 'result.changed' + - '"db_parameter_group_arn" in result' - "'{{ result.db_parameter_group_name | lower }}' == '{{ rds_param_group.name | lower }}'" + - '"description" in result' - 'result.tags == {}' # ============================================================ + - name: test empty parameter group with no arguments changes nothing - CHECK_MODE + rds_param_group: + name: "{{ rds_param_group.name }}" + engine: "{{ rds_param_group.engine }}" + description: "{{ rds_param_group.description }}" + state: present + check_mode: true + register: result + + - name: assert no change when running empty parameter group a second time - CHECK_MODE + assert: + that: + - 'not result.changed' + - name: test empty parameter group with no arguments changes nothing rds_param_group: name: "{{ rds_param_group.name }}" engine: "{{ rds_param_group.engine }}" description: "{{ rds_param_group.description }}" state: present - region: "{{ ec2_region }}" - ec2_access_key: '{{ aws_access_key }}' - ec2_secret_key: '{{ aws_secret_key }}' - security_token: '{{ security_token }}' register: result - ignore_errors: true - name: assert no change when running empty parameter group a second time assert: @@ -56,6 +87,23 @@ - 'not result.changed' # ============================================================ + - name: test adding numeric tag - CHECK_MODE + rds_param_group: + name: "{{ rds_param_group.name }}" + engine: "{{ rds_param_group.engine }}" + description: "{{ rds_param_group.description }}" + state: present + tags: + Environment: test + Test: 123 + check_mode: true + register: result + + - name: adding numeric tag just silently converts - CHECK_MODE + assert: + that: + - 'result.changed' + - name: test adding numeric tag rds_param_group: name: "{{ rds_param_group.name }}" @@ -65,20 +113,39 @@ tags: Environment: test Test: 123 - region: "{{ ec2_region }}" - ec2_access_key: '{{ aws_access_key }}' - ec2_secret_key: '{{ aws_secret_key }}' - security_token: '{{ security_token }}' register: result - ignore_errors: true - name: adding numeric tag just silently converts assert: that: - 'result.changed' - - 'result.tags.Test == "123"' + - '"db_parameter_group_arn" in result' + - "'{{ result.db_parameter_group_name | lower }}' == '{{ rds_param_group.name | lower }}'" + - '"description" in result' + - '"tags" in result' + - result.tags | length == 2 + - result.tags["Environment"] == 'test' + - result.tags["Test"] == '123' # ============================================================ + - name: test tagging existing group - CHECK_MODE + rds_param_group: + name: "{{ rds_param_group.name }}" + engine: "{{ rds_param_group.engine }}" + description: "{{ rds_param_group.description }}" + state: present + tags: + Environment: test + Test: "123" + NewTag: "hello" + check_mode: True + register: result + + - name: assert tagging existing group changes it and adds tags - CHECK_MODE + assert: + that: + - 'result.changed' + - name: test tagging existing group rds_param_group: name: "{{ rds_param_group.name }}" @@ -89,20 +156,48 @@ Environment: test Test: "123" NewTag: "hello" - region: "{{ ec2_region }}" - ec2_access_key: '{{ aws_access_key }}' - ec2_secret_key: '{{ aws_secret_key }}' - security_token: '{{ security_token }}' register: result - ignore_errors: true - name: assert tagging existing group changes it and adds tags assert: that: - 'result.changed' - - 'result.tags.NewTag == "hello"' + - '"db_parameter_group_arn" in result' + - "'{{ result.db_parameter_group_name | lower }}' == '{{ rds_param_group.name | lower }}'" + - '"description" in result' + - '"tags" in result' + - result.tags | length == 3 + - result.tags["Environment"] == 'test' + - result.tags["Test"] == '123' + - result.tags["NewTag"] == 'hello' # ============================================================ + - name: test repeating tagging existing group - CHECK_MODE + rds_param_group: + name: "{{ rds_param_group.name }}" + engine: "{{ rds_param_group.engine }}" + description: "{{ rds_param_group.description }}" + state: present + tags: + Environment: test + Test: "123" + NewTag: "hello" + check_mode: True + register: result + + - name: assert tagging existing group changes it and adds tags - CHECK_MODE + assert: + that: + - 'not result.changed' + - '"db_parameter_group_arn" in result' + - "'{{ result.db_parameter_group_name | lower }}' == '{{ rds_param_group.name | lower }}'" + - '"description" in result' + - '"tags" in result' + - result.tags | length == 3 + - result.tags["Environment"] == 'test' + - result.tags["Test"] == '123' + - result.tags["NewTag"] == 'hello' + - name: test repeating tagging existing group rds_param_group: name: "{{ rds_param_group.name }}" @@ -113,20 +208,39 @@ Environment: test Test: "123" NewTag: "hello" - region: "{{ ec2_region }}" - ec2_access_key: '{{ aws_access_key }}' - ec2_secret_key: '{{ aws_secret_key }}' - security_token: '{{ security_token }}' register: result - ignore_errors: true - name: assert tagging existing group changes it and adds tags assert: that: - 'not result.changed' - - 'result.tags.Test == "123"' + - '"db_parameter_group_arn" in result' + - "'{{ result.db_parameter_group_name | lower }}' == '{{ rds_param_group.name | lower }}'" + - '"description" in result' + - '"tags" in result' + - result.tags | length == 3 + - result.tags["Environment"] == 'test' + - result.tags["Test"] == '123' + - result.tags["NewTag"] == 'hello' # ============================================================ + - name: test deleting tags from existing group - CHECK_MODE + rds_param_group: + name: "{{ rds_param_group.name }}" + engine: "{{ rds_param_group.engine }}" + description: "{{ rds_param_group.description }}" + state: present + tags: + Environment: test + purge_tags: yes + check_mode: True + register: result + + - name: assert removing tags from existing group changes it - CHECK_MODE + assert: + that: + - 'result.changed' + - name: test deleting tags from existing group rds_param_group: name: "{{ rds_param_group.name }}" @@ -135,33 +249,40 @@ state: present tags: Environment: test - region: "{{ ec2_region }}" - ec2_access_key: '{{ aws_access_key }}' - ec2_secret_key: '{{ aws_secret_key }}' - security_token: '{{ security_token }}' purge_tags: yes register: result - ignore_errors: true - name: assert removing tags from existing group changes it assert: that: - 'result.changed' - - 'result.tags.Environment == "test"' - - '"NewTag" not in result.tags' + - '"db_parameter_group_arn" in result' + - "'{{ result.db_parameter_group_name | lower }}' == '{{ rds_param_group.name | lower }}'" + - '"description" in result' + - '"tags" in result' + - result.tags | length == 1 + - result.tags["Environment"] == 'test' # ============================================================ + - name: test state=absent with engine defined (expect changed=true) - CHECK_MODE + rds_param_group: + name: "{{ rds_param_group.name }}" + engine: "{{ rds_param_group.engine }}" + state: absent + check_mode: True + register: result + + - name: assert state=absent with engine defined (expect changed=true) - CHECK_MODE + assert: + that: + - 'result.changed' + - name: test state=absent with engine defined (expect changed=true) rds_param_group: name: "{{ rds_param_group.name }}" engine: "{{ rds_param_group.engine }}" state: absent - region: "{{ ec2_region }}" - ec2_access_key: '{{ aws_access_key }}' - ec2_secret_key: '{{ aws_secret_key }}' - security_token: '{{ security_token }}' register: result - ignore_errors: true - name: assert state=absent with engine defined (expect changed=true) assert: @@ -169,6 +290,28 @@ - 'result.changed' # ============================================================ + - name: test creating group with parameters - CHECK_MODE + rds_param_group: + name: "{{ rds_param_group.name }}" + engine: "{{ rds_param_group.engine }}" + description: "{{ rds_param_group.description }}" + state: present + params: + log_directory: /var/log/postgresql + log_statement: 'all' + log_duration: on + this_param_does_not_exist: oh_no + tags: + Environment: test + Test: "123" + check_mode: True + register: result + + - name: assert creating a new group with parameter changes it - CHECK_MODE + assert: + that: + - 'result.changed' + - name: test creating group with parameters rds_param_group: name: "{{ rds_param_group.name }}" @@ -183,21 +326,44 @@ tags: Environment: test Test: "123" - region: "{{ ec2_region }}" - ec2_access_key: '{{ aws_access_key }}' - ec2_secret_key: '{{ aws_secret_key }}' - security_token: '{{ security_token }}' register: result - ignore_errors: true - name: assert creating a new group with parameter changes it assert: that: - 'result.changed' - - 'result.tags.Test == "123"' + - '"db_parameter_group_arn" in result' + - "'{{ result.db_parameter_group_name | lower }}' == '{{ rds_param_group.name | lower }}'" + - '"description" in result' + - '"tags" in result' + - result.tags | length == 2 + - result.tags["Environment"] == 'test' + - result.tags["Test"] == '123' - 'result.errors|length == 2' # ============================================================ + - name: test repeating group with parameters - CHECK_MODE + rds_param_group: + name: "{{ rds_param_group.name }}" + engine: "{{ rds_param_group.engine }}" + description: "{{ rds_param_group.description }}" + state: present + params: + log_directory: /var/log/postgresql + log_statement: 'all' + log_duration: on + this_param_does_not_exist: oh_no + tags: + Environment: test + Test: "123" + check_mode: True + register: result + + - name: assert repeating group with parameters does not change it - CHECK_MODE + assert: + that: + - 'not result.changed' + - name: test repeating group with parameters rds_param_group: name: "{{ rds_param_group.name }}" @@ -212,32 +378,41 @@ tags: Environment: test Test: "123" - region: "{{ ec2_region }}" - ec2_access_key: '{{ aws_access_key }}' - ec2_secret_key: '{{ aws_secret_key }}' - security_token: '{{ security_token }}' register: result - ignore_errors: true - name: assert repeating group with parameters does not change it assert: that: - 'not result.changed' - - 'result.tags.Test == "123"' + - '"db_parameter_group_arn" in result' + - "'{{ result.db_parameter_group_name | lower }}' == '{{ rds_param_group.name | lower }}'" + - '"description" in result' + - '"tags" in result' + - result.tags | length == 2 + - result.tags["Environment"] == 'test' + - result.tags["Test"] == '123' - 'result.errors|length == 2' # ============================================================ + - name: test state=absent with engine defined (expect changed=true) - CHECK_MODE + rds_param_group: + name: "{{ rds_param_group.name }}" + engine: "{{ rds_param_group.engine }}" + state: absent + check_mode: True + register: result + + - name: assert state=absent with engine defined (expect changed=true) - CHECK_MODE + assert: + that: + - 'result.changed' + - name: test state=absent with engine defined (expect changed=true) rds_param_group: name: "{{ rds_param_group.name }}" engine: "{{ rds_param_group.engine }}" state: absent - region: "{{ ec2_region }}" - ec2_access_key: '{{ aws_access_key }}' - ec2_secret_key: '{{ aws_secret_key }}' - security_token: '{{ security_token }}' register: result - ignore_errors: true - name: assert state=absent with engine defined (expect changed=true) assert: @@ -245,15 +420,25 @@ - 'result.changed' # ============================================================ + - name: test repeating state=absent (expect changed=false) - CHECK_MODE + rds_param_group: + name: "{{ rds_param_group.name }}" + engine: "{{ rds_param_group.engine }}" + state: absent + register: result + check_mode: True + ignore_errors: true + + - name: assert repeating state=absent (expect changed=false) - CHECK_MODE + assert: + that: + - 'not result.changed' + - name: test repeating state=absent (expect changed=false) rds_param_group: name: "{{ rds_param_group.name }}" engine: "{{ rds_param_group.engine }}" state: absent - region: "{{ ec2_region }}" - ec2_access_key: '{{ aws_access_key }}' - ec2_secret_key: '{{ aws_secret_key }}' - security_token: '{{ security_token }}' register: result ignore_errors: true @@ -263,6 +448,21 @@ - 'not result.changed' # ============================================================ + - name: test creating group with more than 20 parameters - CHECK_MODE + rds_param_group: + name: "{{ rds_param_group.name }}" + engine: "{{ rds_param_group.engine }}" + description: "{{ rds_param_group.description }}" + params: "{{ rds_long_param_list }}" + state: present + check_mode: True + register: result + + - name: assert creating a new group with lots of parameter changes it - CHECK_MODE + assert: + that: + - 'result.changed' + - name: test creating group with more than 20 parameters rds_param_group: name: "{{ rds_param_group.name }}" @@ -270,12 +470,7 @@ description: "{{ rds_param_group.description }}" params: "{{ rds_long_param_list }}" state: present - region: "{{ ec2_region }}" - ec2_access_key: '{{ aws_access_key }}' - ec2_secret_key: '{{ aws_secret_key }}' - security_token: '{{ security_token }}' register: result - ignore_errors: true - name: assert creating a new group with lots of parameter changes it assert: @@ -283,19 +478,29 @@ - 'result.changed' # ============================================================ + - name: test creating group with more than 20 parameters - CHECK_MODE + rds_param_group: + name: "{{ rds_param_group.name }}" + engine: "{{ rds_param_group.engine }}" + description: "{{ rds_param_group.description }}" + params: "{{ rds_long_param_list }}" + state: present + check_mode: True + register: result + + - name: assert repeating a group with lots of parameter does not change it - CHECK_MODE + assert: + that: + - 'not result.changed' + - name: test creating group with more than 20 parameters rds_param_group: name: "{{ rds_param_group.name }}" engine: "{{ rds_param_group.engine }}" description: "{{ rds_param_group.description }}" params: "{{ rds_long_param_list }}" - region: "{{ ec2_region }}" state: present - ec2_access_key: '{{ aws_access_key }}' - ec2_secret_key: '{{ aws_secret_key }}' - security_token: '{{ security_token }}' register: result - ignore_errors: true - name: assert repeating a group with lots of parameter does not change it assert: @@ -308,10 +513,6 @@ rds_param_group: name: "{{ rds_param_group.name }}" state: absent - region: "{{ ec2_region }}" - ec2_access_key: '{{ aws_access_key }}' - ec2_secret_key: '{{ aws_secret_key }}' - security_token: '{{ security_token }}' register: result ignore_errors: true