From ac9376a2feb7aaee5f0dfe9aedf73594c8e5eadc Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Mon, 21 Aug 2023 21:44:09 -0700 Subject: [PATCH 01/26] add support for removing cluster from global db --- plugins/modules/rds_cluster.py | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index 187bfbe28e6..94f848cdb4b 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -103,6 +103,10 @@ aliases: - db_name type: str + db_cluster_arn: + description: The Amazon Resource Name (ARN) for the DB cluster. + returned: always + type: str db_cluster_identifier: description: - The DB cluster (lowercase) identifier. The identifier must contain from 1 to 63 letters, numbers, or @@ -305,6 +309,11 @@ aliases: - maintenance_window type: str + remove_from_global_db: + description: + - If set to true, the cluster will be removed from global DB. + type: bool + required: False replication_source_identifier: description: - The Amazon Resource Name (ARN) of the source DB instance or DB cluster if this DB cluster is created as a Read Replica. @@ -463,6 +472,13 @@ engine: aurora-postgresql state: present db_instance_class: 'db.t3.medium' + +- name: Remove a cluster from global DB + amazon.aws.rds_cluster: + db_cluster_identifier: '{{ cluster_id }}' + global_cluster_identifier: '{{ global_cluster_id }}' + db_cluster_arn: arn:aws:rds:us-east-1:123456789000:cluster:my-test-cluster + remove_from_global_db: true """ RETURN = r""" @@ -1102,6 +1118,26 @@ def ensure_present(cluster, parameters, method_name, method_options_name): return changed +def handle_remove_from_global_db(module): + + changed = False + global_cluster_id = module.params.get("global_cluster_identifier") + db_cluster_id = module.params.get("db_cluster_identifier") + db_cluster_arn = module.params.get("db_cluster_arn") + + if module.check_mode: + module.exit_json(changed=True, msg="Would have removed the db cluster from global db.") + + try: + client.remove_from_global_cluster(DbClusterIdentifier=db_cluster_arn, GlobalClusterIdentifier=global_cluster_id) + changed = True + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws(e, msg=f"Failed to remove cluster {db_cluster_id} from global DB cluster {global_cluster_id}.") + + module.exit_json(changed=changed, msg=f"Successfully removed cluster {db_cluster_id} from global DB cluster {global_cluster_id}") + + + def main(): global module global client @@ -1125,6 +1161,7 @@ def main(): backup_retention_period=dict(type="int", default=1), character_set_name=dict(), database_name=dict(aliases=["db_name"]), + db_cluster_arn=dict(type='str'), db_cluster_identifier=dict(required=True, aliases=["cluster_id", "id", "cluster_name"]), db_cluster_parameter_group_name=dict(), db_subnet_group_name=dict(), @@ -1154,6 +1191,7 @@ def main(): port=dict(type="int"), preferred_backup_window=dict(aliases=["backup_window"]), preferred_maintenance_window=dict(aliases=["maintenance_window"]), + remove_from_global_db=dict(type='bool'), replication_source_identifier=dict(aliases=["replication_src_id"]), restore_to_time=dict(), restore_type=dict(choices=["full-copy", "copy-on-write"]), @@ -1205,6 +1243,9 @@ def main(): except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg="Failed to connect to AWS.") + if module.params.get("remove_from_global_db"): + handle_remove_from_global_db(module) + if module.params.get("engine") and module.params["engine"] in ("mysql", "postgres"): module.require_botocore_at_least("1.23.44", reason="to use mysql and postgres engines") if module.params["state"] == "present": From 577e878d4fa78bc9057bdc10c53d2ae68c365b57 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Mon, 21 Aug 2023 21:48:54 -0700 Subject: [PATCH 02/26] add changelogs fragment --- ...05-rds_cluster-add-support-remove-cluster-from-global-db.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelogs/fragments/1705-rds_cluster-add-support-remove-cluster-from-global-db.yml diff --git a/changelogs/fragments/1705-rds_cluster-add-support-remove-cluster-from-global-db.yml b/changelogs/fragments/1705-rds_cluster-add-support-remove-cluster-from-global-db.yml new file mode 100644 index 00000000000..d1d5331eb55 --- /dev/null +++ b/changelogs/fragments/1705-rds_cluster-add-support-remove-cluster-from-global-db.yml @@ -0,0 +1,2 @@ +minor_changes: +- rds_cluster - Add support for removing cluster from global db (https://github.com/ansible-collections/amazon.aws/pull/1705). From c974a8f380a1873f41f30cadbd657c9f6b45bdfb Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Wed, 23 Aug 2023 12:24:35 -0700 Subject: [PATCH 03/26] make linter happy --- plugins/modules/rds_cluster.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index 94f848cdb4b..651701fe6be 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -105,7 +105,6 @@ type: str db_cluster_arn: description: The Amazon Resource Name (ARN) for the DB cluster. - returned: always type: str db_cluster_identifier: description: @@ -1137,7 +1136,6 @@ def handle_remove_from_global_db(module): module.exit_json(changed=changed, msg=f"Successfully removed cluster {db_cluster_id} from global DB cluster {global_cluster_id}") - def main(): global module global client From 03198428acbed2ec84f9c231b2d426d694a03015 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Wed, 23 Aug 2023 13:24:16 -0700 Subject: [PATCH 04/26] make linter happy --- ...er-add-support-remove-cluster-from-global-db.yml | 1 + plugins/modules/rds_cluster.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/changelogs/fragments/1705-rds_cluster-add-support-remove-cluster-from-global-db.yml b/changelogs/fragments/1705-rds_cluster-add-support-remove-cluster-from-global-db.yml index d1d5331eb55..ce9b3df2295 100644 --- a/changelogs/fragments/1705-rds_cluster-add-support-remove-cluster-from-global-db.yml +++ b/changelogs/fragments/1705-rds_cluster-add-support-remove-cluster-from-global-db.yml @@ -1,2 +1,3 @@ +--- minor_changes: - rds_cluster - Add support for removing cluster from global db (https://github.com/ansible-collections/amazon.aws/pull/1705). diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index 651701fe6be..95091bdf4a2 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -1118,7 +1118,6 @@ def ensure_present(cluster, parameters, method_name, method_options_name): def handle_remove_from_global_db(module): - changed = False global_cluster_id = module.params.get("global_cluster_identifier") db_cluster_id = module.params.get("db_cluster_identifier") @@ -1131,9 +1130,13 @@ def handle_remove_from_global_db(module): client.remove_from_global_cluster(DbClusterIdentifier=db_cluster_arn, GlobalClusterIdentifier=global_cluster_id) changed = True except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: - module.fail_json_aws(e, msg=f"Failed to remove cluster {db_cluster_id} from global DB cluster {global_cluster_id}.") + module.fail_json_aws( + e, msg=f"Failed to remove cluster {db_cluster_id} from global DB cluster {global_cluster_id}." + ) - module.exit_json(changed=changed, msg=f"Successfully removed cluster {db_cluster_id} from global DB cluster {global_cluster_id}") + module.exit_json( + changed=changed, msg=f"Successfully removed cluster {db_cluster_id} from global DB cluster {global_cluster_id}" + ) def main(): @@ -1159,7 +1162,7 @@ def main(): backup_retention_period=dict(type="int", default=1), character_set_name=dict(), database_name=dict(aliases=["db_name"]), - db_cluster_arn=dict(type='str'), + db_cluster_arn=dict(type="str"), db_cluster_identifier=dict(required=True, aliases=["cluster_id", "id", "cluster_name"]), db_cluster_parameter_group_name=dict(), db_subnet_group_name=dict(), @@ -1189,7 +1192,7 @@ def main(): port=dict(type="int"), preferred_backup_window=dict(aliases=["backup_window"]), preferred_maintenance_window=dict(aliases=["maintenance_window"]), - remove_from_global_db=dict(type='bool'), + remove_from_global_db=dict(type="bool"), replication_source_identifier=dict(aliases=["replication_src_id"]), restore_to_time=dict(), restore_type=dict(choices=["full-copy", "copy-on-write"]), From bace6a619ebf035fc486011a708446d03a2383fe Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Thu, 24 Aug 2023 12:29:45 -0700 Subject: [PATCH 05/26] add integration test --- .../rds_cluster_modify/defaults/main.yml | 12 +++ .../rds_cluster_modify/tasks/main.yaml | 9 ++ .../tasks/remove_from_global_db.yaml | 93 +++++++++++++++++++ 3 files changed, 114 insertions(+) create mode 100644 tests/integration/targets/rds_cluster_modify/tasks/remove_from_global_db.yaml diff --git a/tests/integration/targets/rds_cluster_modify/defaults/main.yml b/tests/integration/targets/rds_cluster_modify/defaults/main.yml index 073f7a4c000..90e5142b285 100644 --- a/tests/integration/targets/rds_cluster_modify/defaults/main.yml +++ b/tests/integration/targets/rds_cluster_modify/defaults/main.yml @@ -12,3 +12,15 @@ new_cluster_id: ansible-test-cluster-{{ tiny_prefix }}-new new_port: 1155 new_password: test-rds_password-new new_db_parameter_group_name: ansible-test-db-parameter-group-{{ tiny_prefix }}-new + +test_engine: aurora-mysql +test_engine_version: 5.7 +test_instance_class: db.r5.large + +# Global cluster parameters ================================ +test_global_cluster_name: ansible-test-global-{{ tiny_prefix }} + +# Primary cluster parameters ================================ +test_primary_cluster_name: ansible-test-primary-{{ tiny_prefix }} +test_primary_cluster_region: us-east-1 +test_primary_cluster_instance_name: ansible-test-instance-primary-{{ tiny_prefix }} diff --git a/tests/integration/targets/rds_cluster_modify/tasks/main.yaml b/tests/integration/targets/rds_cluster_modify/tasks/main.yaml index 31a20b9a4da..cfb33b57b37 100644 --- a/tests/integration/targets/rds_cluster_modify/tasks/main.yaml +++ b/tests/integration/targets/rds_cluster_modify/tasks/main.yaml @@ -4,7 +4,16 @@ access_key: '{{ aws_access_key }}' secret_key: '{{ aws_secret_key }}' session_token: '{{ security_token | default(omit) }}' + group/amazon.cloud.aws: + region: '{{ aws_region }}' + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + session_token: '{{ security_token | default(omit) }}' block: + + - name: Run tests for testing remove cluster from global db + import_tasks: remove_from_global_db.yaml + - name: Ensure the resource doesn't exist rds_cluster: id: '{{ cluster_id }}' diff --git a/tests/integration/targets/rds_cluster_modify/tasks/remove_from_global_db.yaml b/tests/integration/targets/rds_cluster_modify/tasks/remove_from_global_db.yaml new file mode 100644 index 00000000000..edda693f1cb --- /dev/null +++ b/tests/integration/targets/rds_cluster_modify/tasks/remove_from_global_db.yaml @@ -0,0 +1,93 @@ +--- +- name: Run tests for testing remove cluster from global db + block: + + - name: Create rds global database + amazon.cloud.rds_global_cluster: + global_cluster_identifier: "{{ test_global_cluster_name }}" + engine: "{{ test_engine }}" + engine_version: "{{ test_engine_version }}" + region: "{{ aws_region }}" + state: present + register: create_global_result + + - name: Create a primary cluster for global database + amazon.aws.rds_cluster: + global_cluster_identifier: "{{ test_global_cluster_name }}" + db_cluster_identifier: "{{ test_primary_cluster_name }}" + engine: "{{ test_engine }}" + engine_version: "{{ test_engine_version }}" + username: "{{ username }}" + password: "{{ password }}" + register: create_primary_result + + - name: Get primary cluster info + amazon.aws.rds_cluster: + global_cluster_identifier: "{{ test_global_cluster_name }}" + db_cluster_identifier: "{{ test_primary_cluster_name }}" + register: primary_cluster_info_result + + - name: Get global db info + command: 'aws rds describe-global-clusters --global-cluster-identifier {{ test_global_cluster_name }}' + environment: + AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" + AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" + AWS_SESSION_TOKEN: "{{ security_token | default('') }}" + AWS_DEFAULT_REGION: "{{ aws_region }}" + register: global_cluster_info_result + + - name: convert it to an object + set_fact: + global_cluster_info: "{{ global_cluster_info_result.stdout | from_json }}" + + - name: Assert that primary cluster is a part of global db + assert: + that: + - global_cluster_info.GlobalClusters[0].GlobalClusterMembers[0].DBClusterArn == primary_cluster_info_result.db_cluster_arn + + - name: Remove primary cluster from global db + amazon.aws.rds_cluster: + global_cluster_identifier: '{{ test_global_cluster_name }}' + db_cluster_identifier: '{{ test_primary_cluster_name }}' + db_cluster_arn: "{{ primary_cluster_info_result.db_cluster_arn }}" + remove_from_global_db: true + + - name: Get global db info + command: 'aws rds describe-global-clusters --global-cluster-identifier {{ test_global_cluster_name }}' + environment: + AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" + AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" + AWS_SESSION_TOKEN: "{{ security_token | default('') }}" + AWS_DEFAULT_REGION: "{{ aws_region }}" + register: global_cluster_info_result + + - name: convert it to an object + set_fact: + global_cluster_info: "{{ global_cluster_info_result.stdout | from_json }}" + + - name: Assert that primary cluster is NOT a part of global db + assert: + that: + - global_cluster_info.GlobalClusters[0].GlobalClusterMembers | length == 0 + + always: + - name: Delete primary cluster + amazon.aws.rds_cluster: + db_cluster_identifier: "{{ test_primary_cluster_name }}" + global_cluster_identifier: "{{ test_global_cluster_name }}" + region: "{{ aws_region }}" + engine: "{{ test_engine }}" + engine_version: "{{ test_engine_version }}" + username: "{{ username }}" + password: "{{ password }}" + skip_final_snapshot: true + state: absent + ignore_errors: true + + - name: Delete global db + amazon.cloud.rds_global_cluster: + global_cluster_identifier: "{{ test_global_cluster_name }}" + engine: "{{ test_engine }}" + engine_version: "{{ test_engine_version }}" + state: absent + ignore_errors: true \ No newline at end of file From 1ca22c62a3d80d0ae4f62687b0f8f621de2da151 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Thu, 31 Aug 2023 00:59:29 -0700 Subject: [PATCH 06/26] restructure code logic to make remove_from_global_db a flag and not a isolated feature --- plugins/modules/rds_cluster.py | 42 ++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index 95091bdf4a2..2b0439f3ba2 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -311,6 +311,7 @@ remove_from_global_db: description: - If set to true, the cluster will be removed from global DB. + - Parameters global_cluster_identifier, db_cluster_identifier must be specified when I(remove_from_global_db=true). type: bool required: False replication_source_identifier: @@ -472,12 +473,21 @@ state: present db_instance_class: 'db.t3.medium' -- name: Remove a cluster from global DB +- name: Remove a cluster from global DB (do not delete) amazon.aws.rds_cluster: db_cluster_identifier: '{{ cluster_id }}' global_cluster_identifier: '{{ global_cluster_id }}' - db_cluster_arn: arn:aws:rds:us-east-1:123456789000:cluster:my-test-cluster remove_from_global_db: true + +- name: Remove a cluster from global DB and Delete without creating a final snapshot + amazon.aws.rds_cluster: + engine: aurora + password: "{{ password }}" + username: "{{ username }}" + cluster_id: "{{ cluster_id }}" + skip_final_snapshot: true + remove_from_global_db: true + state: absent """ RETURN = r""" @@ -707,6 +717,8 @@ sample: sg-12345678 """ +import time + try: import botocore except ImportError: @@ -1117,26 +1129,27 @@ def ensure_present(cluster, parameters, method_name, method_options_name): return changed -def handle_remove_from_global_db(module): - changed = False +def handle_remove_from_global_db(module, cluster): global_cluster_id = module.params.get("global_cluster_identifier") db_cluster_id = module.params.get("db_cluster_identifier") - db_cluster_arn = module.params.get("db_cluster_arn") + db_cluster_arn = cluster["DBClusterArn"] if module.check_mode: - module.exit_json(changed=True, msg="Would have removed the db cluster from global db.") + return True try: client.remove_from_global_cluster(DbClusterIdentifier=db_cluster_arn, GlobalClusterIdentifier=global_cluster_id) - changed = True except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws( e, msg=f"Failed to remove cluster {db_cluster_id} from global DB cluster {global_cluster_id}." ) - module.exit_json( - changed=changed, msg=f"Successfully removed cluster {db_cluster_id} from global DB cluster {global_cluster_id}" - ) + # this is needed as even though we have wait_for_cluster_status, it takes a few seconds + # for cluster to change status from 'available' to 'promoting' + time.sleep(15) + wait_for_cluster_status(client, module, db_cluster_id, "cluster_available") + + return True def main(): @@ -1244,9 +1257,6 @@ def main(): except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg="Failed to connect to AWS.") - if module.params.get("remove_from_global_db"): - handle_remove_from_global_db(module) - if module.params.get("engine") and module.params["engine"] in ("mysql", "postgres"): module.require_botocore_at_least("1.23.44", reason="to use mysql and postgres engines") if module.params["state"] == "present": @@ -1287,8 +1297,12 @@ def main(): msg="skip_final_snapshot is False but all of the following are missing: final_snapshot_identifier" ) - parameters = arg_spec_to_rds_params(dict((k, module.params[k]) for k in module.params if k in parameter_options)) changed = False + + if module.params.get("remove_from_global_db"): + changed = handle_remove_from_global_db(module, cluster) + + parameters = arg_spec_to_rds_params(dict((k, module.params[k]) for k in module.params if k in parameter_options)) method_name, method_options_name = get_rds_method_attribute_name(cluster) if method_name: From e1f22ba2470d095c121e565e14ad2ed657c21271 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Thu, 31 Aug 2023 15:09:20 -0700 Subject: [PATCH 07/26] update integration tests as per new logic --- .../rds_cluster_modify/defaults/main.yml | 7 +- .../rds_cluster_modify/tasks/main.yaml | 6 +- .../tasks/remove_from_global_db.yaml | 163 +++++++++++++++++- 3 files changed, 168 insertions(+), 8 deletions(-) diff --git a/tests/integration/targets/rds_cluster_modify/defaults/main.yml b/tests/integration/targets/rds_cluster_modify/defaults/main.yml index 90e5142b285..29e0f650ce4 100644 --- a/tests/integration/targets/rds_cluster_modify/defaults/main.yml +++ b/tests/integration/targets/rds_cluster_modify/defaults/main.yml @@ -22,5 +22,10 @@ test_global_cluster_name: ansible-test-global-{{ tiny_prefix }} # Primary cluster parameters ================================ test_primary_cluster_name: ansible-test-primary-{{ tiny_prefix }} -test_primary_cluster_region: us-east-1 +test_primary_cluster_region: us-west-2 test_primary_cluster_instance_name: ansible-test-instance-primary-{{ tiny_prefix }} + +# Replica cluster parameters ================================ +test_replica_cluster_name: ansible-test-replica-{{ tiny_prefix }} +test_replica_cluster_region: eu-north-1 +test_replica_cluster_instance_name: ansible-test-instance-replica-{{ tiny_prefix }} diff --git a/tests/integration/targets/rds_cluster_modify/tasks/main.yaml b/tests/integration/targets/rds_cluster_modify/tasks/main.yaml index cfb33b57b37..71f34100be0 100644 --- a/tests/integration/targets/rds_cluster_modify/tasks/main.yaml +++ b/tests/integration/targets/rds_cluster_modify/tasks/main.yaml @@ -11,8 +11,10 @@ session_token: '{{ security_token | default(omit) }}' block: - - name: Run tests for testing remove cluster from global db - import_tasks: remove_from_global_db.yaml + # Disabled: Below tests require use of more than 1 region, not supported by CI at the moment + # Tests have been ran, tested, and verified locally on us-west-2 (primary), eu-north-1 (replica) + # - name: Run tests for testing remove cluster from global db + # import_tasks: remove_from_global_db.yaml - name: Ensure the resource doesn't exist rds_cluster: diff --git a/tests/integration/targets/rds_cluster_modify/tasks/remove_from_global_db.yaml b/tests/integration/targets/rds_cluster_modify/tasks/remove_from_global_db.yaml index edda693f1cb..e74037e2169 100644 --- a/tests/integration/targets/rds_cluster_modify/tasks/remove_from_global_db.yaml +++ b/tests/integration/targets/rds_cluster_modify/tasks/remove_from_global_db.yaml @@ -2,15 +2,19 @@ - name: Run tests for testing remove cluster from global db block: + # # Create global db ------------------------------------------------------------------------------- + - name: Create rds global database amazon.cloud.rds_global_cluster: global_cluster_identifier: "{{ test_global_cluster_name }}" engine: "{{ test_engine }}" engine_version: "{{ test_engine_version }}" - region: "{{ aws_region }}" + region: "{{ test_primary_cluster_region }}" state: present register: create_global_result + # Create primary cluster with an instance --------------------------------------------------------------- + - name: Create a primary cluster for global database amazon.aws.rds_cluster: global_cluster_identifier: "{{ test_global_cluster_name }}" @@ -21,6 +25,14 @@ password: "{{ password }}" register: create_primary_result + - name: Create an instance connected to primary cluster + amazon.aws.rds_instance: + db_cluster_identifier: "{{ test_primary_cluster_name }}" + db_instance_identifier: "{{ test_primary_cluster_name }}-instance" + region: "{{ test_primary_cluster_region }}" + engine: "{{ test_engine }}" + db_instance_class: "{{ test_instance_class }}" + - name: Get primary cluster info amazon.aws.rds_cluster: global_cluster_identifier: "{{ test_global_cluster_name }}" @@ -33,7 +45,7 @@ AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" AWS_SESSION_TOKEN: "{{ security_token | default('') }}" - AWS_DEFAULT_REGION: "{{ aws_region }}" + AWS_DEFAULT_REGION: "{{ test_primary_cluster_region }}" register: global_cluster_info_result - name: convert it to an object @@ -45,6 +57,124 @@ that: - global_cluster_info.GlobalClusters[0].GlobalClusterMembers[0].DBClusterArn == primary_cluster_info_result.db_cluster_arn + # Create replica cluster ------------------------------------------------------------------------------- + + - name: Create a replica cluster for global database + amazon.aws.rds_cluster: + global_cluster_identifier: "{{ test_global_cluster_name }}" + db_cluster_identifier: "{{ test_replica_cluster_name }}" + engine: "{{ test_engine }}" + engine_version: "{{ test_engine_version }}" + region: "{{ test_replica_cluster_region }}" + register: create_replica_result + + - name: Get replica cluster info + amazon.aws.rds_cluster: + global_cluster_identifier: "{{ test_global_cluster_name }}" + db_cluster_identifier: "{{ test_replica_cluster_name }}" + region: "{{ test_replica_cluster_region }}" + register: replica_cluster_info_result + + - name: Get global db info + command: 'aws rds describe-global-clusters --global-cluster-identifier {{ test_global_cluster_name }}' + environment: + AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" + AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" + AWS_SESSION_TOKEN: "{{ security_token | default('') }}" + AWS_DEFAULT_REGION: "{{ test_primary_cluster_region }}" + register: global_cluster_info_result + + - name: convert it to an object + set_fact: + global_cluster_info: "{{ global_cluster_info_result.stdout | from_json }}" + + - name: Assert that replica cluster is a part of global db + assert: + that: + - global_cluster_info.GlobalClusters[0].GlobalClusterMembers[1].DBClusterArn == replica_cluster_info_result.db_cluster_arn + + # Test delete on replica cluster part of global db---------------------------------------------------------------- + + - name: Delete DB cluster without final snapshot (fails as its a part of global db) + amazon.aws.rds_cluster: + db_cluster_identifier: "{{ test_replica_cluster_name }}" + global_cluster_identifier: "{{ test_global_cluster_name }}" + region: "{{ test_replica_cluster_region }}" + skip_final_snapshot: true + state: absent + register: delete_replica_cluster_result + ignore_errors: true + + - name: Assert that deletion failed due to cluster being part of global db + assert: + that: + - delete_replica_cluster_result is failed + - delete_replica_cluster_result is not changed + - '"is a part of a global cluster, please remove it from global cluster" in delete_replica_cluster_result.error.message' + + # Test modify replica DB cluster along with removing it from global db------------------------------------------------ + + - name: Remove replica DB cluster from global DB and modify cluster port + amazon.aws.rds_cluster: + db_cluster_identifier: "{{ test_replica_cluster_name }}" + global_cluster_identifier: "{{ test_global_cluster_name }}" + remove_from_global_db: true + state: present + port: 3389 + region: "{{ test_replica_cluster_region }}" + register: modify_port_result + + - name: Get replica cluster info + amazon.aws.rds_cluster: + db_cluster_identifier: "{{ test_replica_cluster_name }}" + region: "{{ test_replica_cluster_region }}" + register: replica_cluster_info_result + + - assert: + that: + - modify_port_result is not failed + - modify_port_result is changed + - replica_cluster_info_result.port == 3389 + + - name: Get global db info + command: 'aws rds describe-global-clusters --global-cluster-identifier {{ test_global_cluster_name }}' + environment: + AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" + AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" + AWS_SESSION_TOKEN: "{{ security_token | default('') }}" + AWS_DEFAULT_REGION: "{{ test_primary_cluster_region }}" + register: global_cluster_info_result + + - name: convert it to an object + set_fact: + global_cluster_info: "{{ global_cluster_info_result.stdout | from_json }}" + - debug: var=global_cluster_info + - name: Assert that replica cluster is NOT a part of global db + assert: + that: + - global_cluster_info.GlobalClusters[0].GlobalClusterMembers | length == 1 + - global_cluster_info.GlobalClusters[0].GlobalClusterMembers[0].DBClusterArn != replica_cluster_info_result.db_cluster_arn + + + # Test delete on replica cluster as NOT a part of global db---------------------------------------------------------------- + + - name: Delete DB cluster without final snapshot (fails as its a part of global db) + amazon.aws.rds_cluster: + db_cluster_identifier: "{{ test_replica_cluster_name }}" + global_cluster_identifier: "{{ test_global_cluster_name }}" + region: "{{ test_replica_cluster_region }}" + skip_final_snapshot: true + state: absent + register: delete_replica_cluster_result + + - name: Assert that deletion failed due to cluster being part of global db + assert: + that: + - delete_replica_cluster_result is not failed + - delete_replica_cluster_result is changed changed + + + # Test remove primary cluster from global db------------------------------------------------------------ - name: Remove primary cluster from global db amazon.aws.rds_cluster: global_cluster_identifier: '{{ test_global_cluster_name }}' @@ -58,7 +188,7 @@ AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" AWS_SESSION_TOKEN: "{{ security_token | default('') }}" - AWS_DEFAULT_REGION: "{{ aws_region }}" + AWS_DEFAULT_REGION: "{{ test_primary_cluster_region }}" register: global_cluster_info_result - name: convert it to an object @@ -70,17 +200,39 @@ that: - global_cluster_info.GlobalClusters[0].GlobalClusterMembers | length == 0 + # Cleanup starts------------------------------------------------------------ + always: + - name: Delete replica cluster + amazon.aws.rds_cluster: + db_cluster_identifier: "{{ test_replica_cluster_name }}" + global_cluster_identifier: "{{ test_global_cluster_name }}" + skip_final_snapshot: true + region: "{{ test_replica_cluster_region }}" + state: absent + ignore_errors: true + + - name: Delete instance connected to primary cluster + amazon.aws.rds_instance: + db_cluster_identifier: "{{ test_primary_cluster_name }}" + db_instance_identifier: "{{ test_primary_cluster_name }}-instance" + engine: "{{ test_engine }}" + db_instance_class: "{{ test_instance_class }}" + skip_final_snapshot: true + region: "{{ test_primary_cluster_region }}" + state: absent + ignore_errors: true + - name: Delete primary cluster amazon.aws.rds_cluster: db_cluster_identifier: "{{ test_primary_cluster_name }}" global_cluster_identifier: "{{ test_global_cluster_name }}" - region: "{{ aws_region }}" engine: "{{ test_engine }}" engine_version: "{{ test_engine_version }}" username: "{{ username }}" password: "{{ password }}" skip_final_snapshot: true + region: "{{ test_primary_cluster_region }}" state: absent ignore_errors: true @@ -89,5 +241,6 @@ global_cluster_identifier: "{{ test_global_cluster_name }}" engine: "{{ test_engine }}" engine_version: "{{ test_engine_version }}" + region: "{{ test_primary_cluster_region }}" state: absent - ignore_errors: true \ No newline at end of file + ignore_errors: true From e168c2deba1389d0a966804766dc1b8f2b50fc58 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Tue, 5 Sep 2023 12:24:19 -0700 Subject: [PATCH 08/26] modified based on review --- plugins/modules/rds_cluster.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index 2b0439f3ba2..385a9326625 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -310,10 +310,11 @@ type: str remove_from_global_db: description: - - If set to true, the cluster will be removed from global DB. - - Parameters global_cluster_identifier, db_cluster_identifier must be specified when I(remove_from_global_db=true). + - If set to C(true), the cluster will be removed from global DB. + - Parameters I(global_cluster_identifier), I(db_cluster_identifier) must be specified when I(remove_from_global_db=true). type: bool required: False + version_added: 6.5.0 replication_source_identifier: description: - The Amazon Resource Name (ARN) of the source DB instance or DB cluster if this DB cluster is created as a Read Replica. @@ -1300,7 +1301,8 @@ def main(): changed = False if module.params.get("remove_from_global_db"): - changed = handle_remove_from_global_db(module, cluster) + if module.params.get("engine") in ["aurora", "aurora-mysql", "aurora-postgresql"]: + changed = handle_remove_from_global_db(module, cluster) parameters = arg_spec_to_rds_params(dict((k, module.params[k]) for k in module.params if k in parameter_options)) method_name, method_options_name = get_rds_method_attribute_name(cluster) From fd0a0731ea3072df2f75ee306569804ab7d609b4 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Tue, 5 Sep 2023 13:43:14 -0700 Subject: [PATCH 09/26] wait for cluster state to be available only when wait=true --- plugins/modules/rds_cluster.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index 385a9326625..1979520627b 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -1148,7 +1148,9 @@ def handle_remove_from_global_db(module, cluster): # this is needed as even though we have wait_for_cluster_status, it takes a few seconds # for cluster to change status from 'available' to 'promoting' time.sleep(15) - wait_for_cluster_status(client, module, db_cluster_id, "cluster_available") + + if module.params.get('wait'): + wait_for_cluster_status(client, module, db_cluster_id, "cluster_available") return True From eedaa256c9c39bfb745f4f06b95261772e6c12d2 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Wed, 6 Sep 2023 12:54:16 -0700 Subject: [PATCH 10/26] replace static 'sleep' with custom waiter to wait for status to change from 'available' to 'promoting' --- plugins/module_utils/rds.py | 2 ++ plugins/module_utils/waiters.py | 18 ++++++++++++++++++ plugins/modules/rds_cluster.py | 17 +++++++++-------- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/plugins/module_utils/rds.py b/plugins/module_utils/rds.py index d4fb5380748..5141b5ad62d 100644 --- a/plugins/module_utils/rds.py +++ b/plugins/module_utils/rds.py @@ -123,6 +123,8 @@ def get_rds_method_attribute(method_name, module): waiter = "role_disassociated" elif method_name == "promote_read_replica": waiter = "read_replica_promoted" + elif method_name == "db_cluster_promoting": + waiter = "db_cluster_promoting" else: waiter = "db_instance_available" # Handle retry codes diff --git a/plugins/module_utils/waiters.py b/plugins/module_utils/waiters.py index f8a3b69c117..0076feb16ff 100644 --- a/plugins/module_utils/waiters.py +++ b/plugins/module_utils/waiters.py @@ -547,6 +547,19 @@ rds_data = { "version": 2, "waiters": { + "DBClusterPromoting": { + "delay": 2, + "maxAttempts": 10, + "operation": "DescribeDBClusters", + "acceptors": [ + { + "state": "success", + "matcher": "pathAll", + "argument": "DBClusters[].Status", + "expected": "promoting", + }, + ], + }, "DBInstanceStopped": { "delay": 20, "maxAttempts": 60, @@ -911,6 +924,11 @@ def route53_model(name): elbv2_model("LoadBalancersDeleted"), core_waiter.NormalizedOperationMethod(elbv2.describe_load_balancers), ), + ("RDS", "db_cluster_promoting"): lambda rds: core_waiter.Waiter( + "db_cluster_promoting", + rds_model("DBClusterPromoting"), + core_waiter.NormalizedOperationMethod(rds.describe_db_clusters), + ), ("RDS", "db_instance_stopped"): lambda rds: core_waiter.Waiter( "db_instance_stopped", rds_model("DBInstanceStopped"), diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index 1979520627b..bb9ea577300 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -312,6 +312,8 @@ description: - If set to C(true), the cluster will be removed from global DB. - Parameters I(global_cluster_identifier), I(db_cluster_identifier) must be specified when I(remove_from_global_db=true). + - When performing other modifications along with I(remove_grom_global_db=true), make sure to have I(wait=true) as DB cluster + isn't available for modification while promoting. type: bool required: False version_added: 6.5.0 @@ -488,6 +490,7 @@ cluster_id: "{{ cluster_id }}" skip_final_snapshot: true remove_from_global_db: true + wait: true state: absent """ @@ -718,8 +721,6 @@ sample: sg-12345678 """ -import time - try: import botocore except ImportError: @@ -1145,12 +1146,12 @@ def handle_remove_from_global_db(module, cluster): e, msg=f"Failed to remove cluster {db_cluster_id} from global DB cluster {global_cluster_id}." ) - # this is needed as even though we have wait_for_cluster_status, it takes a few seconds - # for cluster to change status from 'available' to 'promoting' - time.sleep(15) + # wait for cluster to change status from 'available' to 'promoting' + wait_for_cluster_status(client, module, db_cluster_id, "db_cluster_promoting") - if module.params.get('wait'): - wait_for_cluster_status(client, module, db_cluster_id, "cluster_available") + # if wait=true, wait for db cluster remove from global db operation to complete + if module.params.get("wait"): + wait_for_cluster_status(client, module, db_cluster_id, "cluster_available") return True @@ -1303,7 +1304,7 @@ def main(): changed = False if module.params.get("remove_from_global_db"): - if module.params.get("engine") in ["aurora", "aurora-mysql", "aurora-postgresql"]: + if cluster["Engine"] in ["aurora", "aurora-mysql", "aurora-postgresql"]: changed = handle_remove_from_global_db(module, cluster) parameters = arg_spec_to_rds_params(dict((k, module.params[k]) for k in module.params if k in parameter_options)) From 1c641a8f0cc53c87dbf32d690f8190064afa832d Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Wed, 6 Sep 2023 12:57:15 -0700 Subject: [PATCH 11/26] add amazon.cloud to test requirements.yml --- tests/integration/requirements.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/requirements.yml b/tests/integration/requirements.yml index df4d6171dc1..fa69c3272a8 100644 --- a/tests/integration/requirements.yml +++ b/tests/integration/requirements.yml @@ -2,3 +2,4 @@ collections: - ansible.windows - ansible.utils # ipv6 filter +- amazon.cloud # used by integration tests - rds_cluster_modify From 53fcb46cdeb3875f798db5db4604719127b58666 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Fri, 8 Sep 2023 00:47:19 -0700 Subject: [PATCH 12/26] update logic for wait when remove_from_global_db for primary and replica db --- plugins/modules/rds_cluster.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index bb9ea577300..ab82d19c0f5 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -312,7 +312,7 @@ description: - If set to C(true), the cluster will be removed from global DB. - Parameters I(global_cluster_identifier), I(db_cluster_identifier) must be specified when I(remove_from_global_db=true). - - When performing other modifications along with I(remove_grom_global_db=true), make sure to have I(wait=true) as DB cluster + - When performing other modifications along with I(remove_grom_global_db=true), must set I(wait=true) as DB cluster isn't available for modification while promoting. type: bool required: False @@ -1146,8 +1146,10 @@ def handle_remove_from_global_db(module, cluster): e, msg=f"Failed to remove cluster {db_cluster_id} from global DB cluster {global_cluster_id}." ) - # wait for cluster to change status from 'available' to 'promoting' - wait_for_cluster_status(client, module, db_cluster_id, "db_cluster_promoting") + # for replica cluster - wait for cluster to change status from 'available' to 'promoting' + # only replica/secondary clusters have "GlobalWriteForwardingStatus" field + if "GlobalWriteForwardingStatus" in cluster: + wait_for_cluster_status(client, module, db_cluster_id, "db_cluster_promoting") # if wait=true, wait for db cluster remove from global db operation to complete if module.params.get("wait"): From 492fdcd68b0cd5825d196cd7225391d5b7c3500b Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Fri, 8 Sep 2023 00:50:14 -0700 Subject: [PATCH 13/26] remove unused new param added earlier --- plugins/modules/rds_cluster.py | 4 ---- .../targets/rds_cluster_modify/tasks/main.yaml | 13 ++++--------- .../tasks/remove_from_global_db.yaml | 14 ++++++-------- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index ab82d19c0f5..23ad2644937 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -103,9 +103,6 @@ aliases: - db_name type: str - db_cluster_arn: - description: The Amazon Resource Name (ARN) for the DB cluster. - type: str db_cluster_identifier: description: - The DB cluster (lowercase) identifier. The identifier must contain from 1 to 63 letters, numbers, or @@ -1181,7 +1178,6 @@ def main(): backup_retention_period=dict(type="int", default=1), character_set_name=dict(), database_name=dict(aliases=["db_name"]), - db_cluster_arn=dict(type="str"), db_cluster_identifier=dict(required=True, aliases=["cluster_id", "id", "cluster_name"]), db_cluster_parameter_group_name=dict(), db_subnet_group_name=dict(), diff --git a/tests/integration/targets/rds_cluster_modify/tasks/main.yaml b/tests/integration/targets/rds_cluster_modify/tasks/main.yaml index 71f34100be0..e3b9687075d 100644 --- a/tests/integration/targets/rds_cluster_modify/tasks/main.yaml +++ b/tests/integration/targets/rds_cluster_modify/tasks/main.yaml @@ -4,17 +4,12 @@ access_key: '{{ aws_access_key }}' secret_key: '{{ aws_secret_key }}' session_token: '{{ security_token | default(omit) }}' - group/amazon.cloud.aws: - region: '{{ aws_region }}' - aws_access_key: '{{ aws_access_key }}' - aws_secret_key: '{{ aws_secret_key }}' - session_token: '{{ security_token | default(omit) }}' block: - # Disabled: Below tests require use of more than 1 region, not supported by CI at the moment - # Tests have been ran, tested, and verified locally on us-west-2 (primary), eu-north-1 (replica) - # - name: Run tests for testing remove cluster from global db - # import_tasks: remove_from_global_db.yaml + # Disabled: Below tests require use of more than 1 region, not supported by CI at the moment + # Tests have been ran, tested, and verified locally on us-west-2 (primary), eu-north-1 (replica) + # - name: Run tests for testing remove cluster from global db + # import_tasks: remove_from_global_db.yaml - name: Ensure the resource doesn't exist rds_cluster: diff --git a/tests/integration/targets/rds_cluster_modify/tasks/remove_from_global_db.yaml b/tests/integration/targets/rds_cluster_modify/tasks/remove_from_global_db.yaml index e74037e2169..159850a9b35 100644 --- a/tests/integration/targets/rds_cluster_modify/tasks/remove_from_global_db.yaml +++ b/tests/integration/targets/rds_cluster_modify/tasks/remove_from_global_db.yaml @@ -2,7 +2,7 @@ - name: Run tests for testing remove cluster from global db block: - # # Create global db ------------------------------------------------------------------------------- + # Create global db ------------------------------------------------------------------------------- - name: Create rds global database amazon.cloud.rds_global_cluster: @@ -148,17 +148,16 @@ - name: convert it to an object set_fact: global_cluster_info: "{{ global_cluster_info_result.stdout | from_json }}" - - debug: var=global_cluster_info + - name: Assert that replica cluster is NOT a part of global db assert: that: - global_cluster_info.GlobalClusters[0].GlobalClusterMembers | length == 1 - global_cluster_info.GlobalClusters[0].GlobalClusterMembers[0].DBClusterArn != replica_cluster_info_result.db_cluster_arn - # Test delete on replica cluster as NOT a part of global db---------------------------------------------------------------- - - name: Delete DB cluster without final snapshot (fails as its a part of global db) + - name: Delete replica cluster amazon.aws.rds_cluster: db_cluster_identifier: "{{ test_replica_cluster_name }}" global_cluster_identifier: "{{ test_global_cluster_name }}" @@ -167,12 +166,11 @@ state: absent register: delete_replica_cluster_result - - name: Assert that deletion failed due to cluster being part of global db + - name: Assert that replica cluster deletion succeeded assert: that: - delete_replica_cluster_result is not failed - - delete_replica_cluster_result is changed changed - + - delete_replica_cluster_result is changed # Test remove primary cluster from global db------------------------------------------------------------ - name: Remove primary cluster from global db @@ -198,7 +196,7 @@ - name: Assert that primary cluster is NOT a part of global db assert: that: - - global_cluster_info.GlobalClusters[0].GlobalClusterMembers | length == 0 + - global_cluster_info.GlobalClusters[0].GlobalClusterMembers | length == 0 # Cleanup starts------------------------------------------------------------ From ba071bc5d07725fb1ef2180ef0e7458f2efa2302 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Mon, 11 Sep 2023 12:41:36 -0700 Subject: [PATCH 14/26] change engine version to fix integration test failure --- tests/integration/targets/rds_cluster_modify/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/targets/rds_cluster_modify/defaults/main.yml b/tests/integration/targets/rds_cluster_modify/defaults/main.yml index 29e0f650ce4..e52c833c84f 100644 --- a/tests/integration/targets/rds_cluster_modify/defaults/main.yml +++ b/tests/integration/targets/rds_cluster_modify/defaults/main.yml @@ -14,7 +14,7 @@ new_password: test-rds_password-new new_db_parameter_group_name: ansible-test-db-parameter-group-{{ tiny_prefix }}-new test_engine: aurora-mysql -test_engine_version: 5.7 +test_engine_version: 8.0 test_instance_class: db.r5.large # Global cluster parameters ================================ From 6d5fffbd387ac841a7e2edbd072b5447fa5ffbeb Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Mon, 11 Sep 2023 14:30:17 -0700 Subject: [PATCH 15/26] doc fix, add required_if for remove_from_global_db --- plugins/modules/rds_cluster.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index 23ad2644937..afc5601f463 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -309,7 +309,7 @@ description: - If set to C(true), the cluster will be removed from global DB. - Parameters I(global_cluster_identifier), I(db_cluster_identifier) must be specified when I(remove_from_global_db=true). - - When performing other modifications along with I(remove_grom_global_db=true), must set I(wait=true) as DB cluster + - When performing other modifications along with I(remove_from_global_db=true),, must set I(wait=true) as DB cluster isn't available for modification while promoting. type: bool required: False @@ -1244,6 +1244,7 @@ def main(): required_if=[ ["creation_source", "snapshot", ["snapshot_identifier", "engine"]], ["creation_source", "s3", required_by_s3_creation_source], + ["remove_from_global_db", True, ["global_cluster_identifier", "db_cluster_identifier"]], ], mutually_exclusive=[ ["s3_bucket_name", "source_db_cluster_identifier", "snapshot_identifier"], From 8b19ae441b6d342e70ac55840a888b480f44dd4a Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Mon, 11 Sep 2023 14:32:47 -0700 Subject: [PATCH 16/26] change engine version to fix integration test failure --- tests/integration/targets/rds_cluster_modify/tasks/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/targets/rds_cluster_modify/tasks/main.yaml b/tests/integration/targets/rds_cluster_modify/tasks/main.yaml index e3b9687075d..d7118807fe8 100644 --- a/tests/integration/targets/rds_cluster_modify/tasks/main.yaml +++ b/tests/integration/targets/rds_cluster_modify/tasks/main.yaml @@ -183,7 +183,7 @@ - name: Create DB cluster parameter group if not exists command: aws rds create-db-cluster-parameter-group --db-cluster-parameter-group-name - {{ new_db_parameter_group_name }} --db-parameter-group-family aurora-mysql5.7 --description + {{ new_db_parameter_group_name }} --db-parameter-group-family aurora-mysql8.0 --description "Test DB cluster parameter group" environment: AWS_ACCESS_KEY_ID: '{{ aws_access_key }}' From d5e650d1aa89e33f8d5afb54cb77a99fcde84eff Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Tue, 12 Sep 2023 16:02:11 -0700 Subject: [PATCH 17/26] fix integration tests --- plugins/module_utils/waiters.py | 2 +- .../tasks/remove_from_global_db.yaml | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/plugins/module_utils/waiters.py b/plugins/module_utils/waiters.py index 0076feb16ff..86bde6b630b 100644 --- a/plugins/module_utils/waiters.py +++ b/plugins/module_utils/waiters.py @@ -549,7 +549,7 @@ "waiters": { "DBClusterPromoting": { "delay": 2, - "maxAttempts": 10, + "maxAttempts": 30, "operation": "DescribeDBClusters", "acceptors": [ { diff --git a/tests/integration/targets/rds_cluster_modify/tasks/remove_from_global_db.yaml b/tests/integration/targets/rds_cluster_modify/tasks/remove_from_global_db.yaml index 159850a9b35..984c14ff381 100644 --- a/tests/integration/targets/rds_cluster_modify/tasks/remove_from_global_db.yaml +++ b/tests/integration/targets/rds_cluster_modify/tasks/remove_from_global_db.yaml @@ -19,6 +19,7 @@ amazon.aws.rds_cluster: global_cluster_identifier: "{{ test_global_cluster_name }}" db_cluster_identifier: "{{ test_primary_cluster_name }}" + region: "{{ test_primary_cluster_region }}" engine: "{{ test_engine }}" engine_version: "{{ test_engine_version }}" username: "{{ username }}" @@ -34,9 +35,9 @@ db_instance_class: "{{ test_instance_class }}" - name: Get primary cluster info - amazon.aws.rds_cluster: - global_cluster_identifier: "{{ test_global_cluster_name }}" + amazon.aws.rds_cluster_info: db_cluster_identifier: "{{ test_primary_cluster_name }}" + region: "{{ test_primary_cluster_region }}" register: primary_cluster_info_result - name: Get global db info @@ -55,7 +56,7 @@ - name: Assert that primary cluster is a part of global db assert: that: - - global_cluster_info.GlobalClusters[0].GlobalClusterMembers[0].DBClusterArn == primary_cluster_info_result.db_cluster_arn + - global_cluster_info.GlobalClusters[0].GlobalClusterMembers[0].DBClusterArn == primary_cluster_info_result.clusters[0].db_cluster_arn # Create replica cluster ------------------------------------------------------------------------------- @@ -64,13 +65,12 @@ global_cluster_identifier: "{{ test_global_cluster_name }}" db_cluster_identifier: "{{ test_replica_cluster_name }}" engine: "{{ test_engine }}" - engine_version: "{{ test_engine_version }}" + engine_version: "{{ global_cluster_info.GlobalClusters[0].EngineVersion }}" # replica cluster engine version needs to be exact same as global db engine version region: "{{ test_replica_cluster_region }}" register: create_replica_result - name: Get replica cluster info - amazon.aws.rds_cluster: - global_cluster_identifier: "{{ test_global_cluster_name }}" + amazon.aws.rds_cluster_info: db_cluster_identifier: "{{ test_replica_cluster_name }}" region: "{{ test_replica_cluster_region }}" register: replica_cluster_info_result @@ -91,7 +91,7 @@ - name: Assert that replica cluster is a part of global db assert: that: - - global_cluster_info.GlobalClusters[0].GlobalClusterMembers[1].DBClusterArn == replica_cluster_info_result.db_cluster_arn + - global_cluster_info.GlobalClusters[0].GlobalClusterMembers[1].DBClusterArn == replica_cluster_info_result.clusters[0].db_cluster_arn # Test delete on replica cluster part of global db---------------------------------------------------------------- @@ -125,7 +125,7 @@ register: modify_port_result - name: Get replica cluster info - amazon.aws.rds_cluster: + amazon.aws.rds_cluster_info: db_cluster_identifier: "{{ test_replica_cluster_name }}" region: "{{ test_replica_cluster_region }}" register: replica_cluster_info_result @@ -134,7 +134,7 @@ that: - modify_port_result is not failed - modify_port_result is changed - - replica_cluster_info_result.port == 3389 + - replica_cluster_info_result.clusters[0].port == 3389 - name: Get global db info command: 'aws rds describe-global-clusters --global-cluster-identifier {{ test_global_cluster_name }}' @@ -153,7 +153,7 @@ assert: that: - global_cluster_info.GlobalClusters[0].GlobalClusterMembers | length == 1 - - global_cluster_info.GlobalClusters[0].GlobalClusterMembers[0].DBClusterArn != replica_cluster_info_result.db_cluster_arn + - global_cluster_info.GlobalClusters[0].GlobalClusterMembers[0].DBClusterArn != replica_cluster_info_result.clusters[0].db_cluster_arn # Test delete on replica cluster as NOT a part of global db---------------------------------------------------------------- @@ -177,7 +177,7 @@ amazon.aws.rds_cluster: global_cluster_identifier: '{{ test_global_cluster_name }}' db_cluster_identifier: '{{ test_primary_cluster_name }}' - db_cluster_arn: "{{ primary_cluster_info_result.db_cluster_arn }}" + region: "{{ test_primary_cluster_region }}" remove_from_global_db: true - name: Get global db info From cc7091e7f4dbec2292c419ae92530e4a3ae3c788 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Wed, 13 Sep 2023 14:41:02 -0700 Subject: [PATCH 18/26] add another example --- plugins/modules/rds_cluster.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index afc5601f463..905a9ce92fd 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -489,6 +489,15 @@ remove_from_global_db: true wait: true state: absent + +- name: Remove replica DB cluster from global DB and modify cluster port + amazon.aws.rds_cluster: + db_cluster_identifier: "{{ replica_cluster_name }}" + global_cluster_identifier: "{{ global_cluster_name }}" + remove_from_global_db: true + state: present + port: 3389 + region: "{{ replica_cluster_region }}" """ RETURN = r""" From b327e09c5e23b00de25bf3019bc0a59858d18213 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Tue, 19 Sep 2023 16:46:01 -0700 Subject: [PATCH 19/26] update logic to allow update without wait on remove_from_global_db --- plugins/modules/rds_cluster.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index 905a9ce92fd..49f6452cdb3 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -309,8 +309,6 @@ description: - If set to C(true), the cluster will be removed from global DB. - Parameters I(global_cluster_identifier), I(db_cluster_identifier) must be specified when I(remove_from_global_db=true). - - When performing other modifications along with I(remove_from_global_db=true),, must set I(wait=true) as DB cluster - isn't available for modification while promoting. type: bool required: False version_added: 6.5.0 @@ -490,14 +488,24 @@ wait: true state: absent -- name: Remove replica DB cluster from global DB and modify cluster port +- name: Update cluster port and WAIT for remove secondary DB cluster from global DB to complete amazon.aws.rds_cluster: - db_cluster_identifier: "{{ replica_cluster_name }}" + db_cluster_identifier: "{{ secondary_cluster_name }}" global_cluster_identifier: "{{ global_cluster_name }}" remove_from_global_db: true state: present port: 3389 - region: "{{ replica_cluster_region }}" + region: "{{ secondary_cluster_region }}" + +- name: Update cluster port and DO NOT WAIT for remove secondary DB cluster from global DB to complete + amazon.aws.rds_cluster: + db_cluster_identifier: "{{ secondary_cluster_name }}" + global_cluster_identifier: "{{ global_cluster_name }}" + remove_from_global_db: true + state: present + port: 3389 + region: "{{ secondary_cluster_region }}" + wait: false """ RETURN = r""" @@ -1311,15 +1319,15 @@ def main(): changed = False - if module.params.get("remove_from_global_db"): - if cluster["Engine"] in ["aurora", "aurora-mysql", "aurora-postgresql"]: - changed = handle_remove_from_global_db(module, cluster) - parameters = arg_spec_to_rds_params(dict((k, module.params[k]) for k in module.params if k in parameter_options)) method_name, method_options_name = get_rds_method_attribute_name(cluster) if method_name: if method_name == "delete_db_cluster": + if module.params.get("remove_from_global_db"): + if cluster["Engine"] in ["aurora", "aurora-mysql", "aurora-postgresql"]: + changed = handle_remove_from_global_db(module, cluster) + call_method(client, module, method_name, eval(method_options_name)(parameters)) changed = True else: @@ -1330,6 +1338,10 @@ def main(): else: cluster_id = module.params["db_cluster_identifier"] + if module.params.get("remove_from_global_db"): + if cluster["Engine"] in ["aurora", "aurora-mysql", "aurora-postgresql"]: + changed = handle_remove_from_global_db(module, cluster) + result = camel_dict_to_snake_dict(get_cluster(cluster_id)) if result: From d5ad59e57dad774f2d2c23aa55b5a411700b89fb Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Tue, 19 Sep 2023 17:03:25 -0700 Subject: [PATCH 20/26] sanity fix --- plugins/modules/rds_cluster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index 49f6452cdb3..7dc241080f6 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -1325,8 +1325,8 @@ def main(): if method_name: if method_name == "delete_db_cluster": if module.params.get("remove_from_global_db"): - if cluster["Engine"] in ["aurora", "aurora-mysql", "aurora-postgresql"]: - changed = handle_remove_from_global_db(module, cluster) + if cluster["Engine"] in ["aurora", "aurora-mysql", "aurora-postgresql"]: + changed = handle_remove_from_global_db(module, cluster) call_method(client, module, method_name, eval(method_options_name)(parameters)) changed = True From 6096c932d7b976e12472c34b66a624bf77529a30 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Tue, 19 Sep 2023 20:03:39 -0700 Subject: [PATCH 21/26] minor fix --- plugins/modules/rds_cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index 7dc241080f6..06297e9e283 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -1324,7 +1324,7 @@ def main(): if method_name: if method_name == "delete_db_cluster": - if module.params.get("remove_from_global_db"): + if cluster and module.params.get("remove_from_global_db"): if cluster["Engine"] in ["aurora", "aurora-mysql", "aurora-postgresql"]: changed = handle_remove_from_global_db(module, cluster) From 55b096ef5ed01cd218543e8e4821ee589c457440 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Tue, 19 Sep 2023 20:16:37 -0700 Subject: [PATCH 22/26] minor fix --- plugins/modules/rds_cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index 06297e9e283..d134d4d38f2 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -1338,7 +1338,7 @@ def main(): else: cluster_id = module.params["db_cluster_identifier"] - if module.params.get("remove_from_global_db"): + if cluster and module.params.get("remove_from_global_db"): if cluster["Engine"] in ["aurora", "aurora-mysql", "aurora-postgresql"]: changed = handle_remove_from_global_db(module, cluster) From 0e748d8f27cd920ab5d3a2febd2310f1803dd10e Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Thu, 21 Sep 2023 10:35:58 -0700 Subject: [PATCH 23/26] modify as suggested --- plugins/modules/rds_cluster.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index d134d4d38f2..f698d3b3093 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -1340,7 +1340,9 @@ def main(): if cluster and module.params.get("remove_from_global_db"): if cluster["Engine"] in ["aurora", "aurora-mysql", "aurora-postgresql"]: - changed = handle_remove_from_global_db(module, cluster) + if changed: + wait_for_cluster_status(client, module, cluster_id, "cluster_available") + changed |= handle_remove_from_global_db(module, cluster) result = camel_dict_to_snake_dict(get_cluster(cluster_id)) From 3d4b5230718acd7c5aa9d91cb2aeb61e03d14c54 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Tue, 26 Sep 2023 11:37:31 -0700 Subject: [PATCH 24/26] increase wait time and retries --- plugins/module_utils/waiters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/module_utils/waiters.py b/plugins/module_utils/waiters.py index 86bde6b630b..81c9789617e 100644 --- a/plugins/module_utils/waiters.py +++ b/plugins/module_utils/waiters.py @@ -548,8 +548,8 @@ "version": 2, "waiters": { "DBClusterPromoting": { - "delay": 2, - "maxAttempts": 30, + "delay": 5, + "maxAttempts": 60, "operation": "DescribeDBClusters", "acceptors": [ { From 3fa1d7ad34048179de933d5c67a648202b9fea8b Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Tue, 26 Sep 2023 14:35:49 -0700 Subject: [PATCH 25/26] fix edge case --- plugins/modules/rds_cluster.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index f698d3b3093..a8d8bcd4007 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -1333,17 +1333,17 @@ def main(): else: changed |= ensure_present(cluster, parameters, method_name, method_options_name) + if cluster and module.params.get("remove_from_global_db"): + if cluster["Engine"] in ["aurora", "aurora-mysql", "aurora-postgresql"]: + if changed: + wait_for_cluster_status(client, module, cluster_id, "cluster_available") + changed |= handle_remove_from_global_db(module, cluster) + if not module.check_mode and module.params["new_db_cluster_identifier"] and module.params["apply_immediately"]: cluster_id = module.params["new_db_cluster_identifier"] else: cluster_id = module.params["db_cluster_identifier"] - if cluster and module.params.get("remove_from_global_db"): - if cluster["Engine"] in ["aurora", "aurora-mysql", "aurora-postgresql"]: - if changed: - wait_for_cluster_status(client, module, cluster_id, "cluster_available") - changed |= handle_remove_from_global_db(module, cluster) - result = camel_dict_to_snake_dict(get_cluster(cluster_id)) if result: From 7844a8f7347f332b06417617f9d4e769c97913c8 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Tue, 26 Sep 2023 16:07:07 -0700 Subject: [PATCH 26/26] minor fix --- plugins/modules/rds_cluster.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/modules/rds_cluster.py b/plugins/modules/rds_cluster.py index a8d8bcd4007..7378bd86d48 100644 --- a/plugins/modules/rds_cluster.py +++ b/plugins/modules/rds_cluster.py @@ -1333,17 +1333,17 @@ def main(): else: changed |= ensure_present(cluster, parameters, method_name, method_options_name) - if cluster and module.params.get("remove_from_global_db"): - if cluster["Engine"] in ["aurora", "aurora-mysql", "aurora-postgresql"]: - if changed: - wait_for_cluster_status(client, module, cluster_id, "cluster_available") - changed |= handle_remove_from_global_db(module, cluster) - if not module.check_mode and module.params["new_db_cluster_identifier"] and module.params["apply_immediately"]: cluster_id = module.params["new_db_cluster_identifier"] else: cluster_id = module.params["db_cluster_identifier"] + if cluster_id and get_cluster(cluster_id) and module.params.get("remove_from_global_db"): + if cluster["Engine"] in ["aurora", "aurora-mysql", "aurora-postgresql"]: + if changed: + wait_for_cluster_status(client, module, cluster_id, "cluster_available") + changed |= handle_remove_from_global_db(module, cluster) + result = camel_dict_to_snake_dict(get_cluster(cluster_id)) if result: