From 1879a628fd83798e87af2e578fb34bef8996e19c Mon Sep 17 00:00:00 2001 From: abikouo <79859644+abikouo@users.noreply.github.com> Date: Thu, 24 Jun 2021 12:23:16 +0200 Subject: [PATCH] ec2_vol support MultiAttach disk (#365) ec2_vol support MultiAttach disk SUMMARY new parameter multi_attach to support MultiAttach on disk creation/update #225 ISSUE TYPE Feature Pull Request COMPONENT NAME ec2_vol ADDITIONAL INFORMATION Reviewed-by: Markus Bergholz Reviewed-by: None Reviewed-by: Mark Chappell --- ...362-ec2_vol-add-multi-attach-parameter.yml | 4 + plugins/modules/ec2_vol.py | 91 ++++++---- plugins/modules/ec2_vol_info.py | 25 ++- .../targets/ec2_vol/tasks/tests.yml | 161 ++++++++++++++---- 4 files changed, 208 insertions(+), 73 deletions(-) create mode 100644 changelogs/fragments/362-ec2_vol-add-multi-attach-parameter.yml diff --git a/changelogs/fragments/362-ec2_vol-add-multi-attach-parameter.yml b/changelogs/fragments/362-ec2_vol-add-multi-attach-parameter.yml new file mode 100644 index 00000000000..c029a5bcf4f --- /dev/null +++ b/changelogs/fragments/362-ec2_vol-add-multi-attach-parameter.yml @@ -0,0 +1,4 @@ +minor_changes: +- ec2_vol - add parameter ``multi_attach`` to support Multi-Attach on volume creation/update (https://github.com/ansible-collections/amazon.aws/pull/362). +breaking_changes: +- ec2_vol_info - return ``attachment_set`` is now a list of attachments with Multi-Attach support on disk. (https://github.com/ansible-collections/amazon.aws/pull/362). diff --git a/plugins/modules/ec2_vol.py b/plugins/modules/ec2_vol.py index 400377acbd8..d459e4c547b 100644 --- a/plugins/modules/ec2_vol.py +++ b/plugins/modules/ec2_vol.py @@ -104,6 +104,13 @@ - Requires at least botocore version 1.19.27. type: int version_added: 1.4.0 + multi_attach: + description: + - If set to C(yes), Multi-Attach will be enabled when creating the volume. + - When you create a new volume, Multi-Attach is disabled by default. + - This parameter is supported with io1 and io2 volumes only. + type: bool + version_added: 2.0.0 author: "Lester Wade (@lwade)" extends_documentation_fragment: - amazon.aws.aws @@ -189,6 +196,14 @@ volume_type: gp2 device_name: /dev/xvdf +# Create new volume with multi-attach enabled +- amazon.aws.ec2_vol: + zone: XXXXXX + multi_attach: true + volume_size: 4 + volume_type: io1 + iops: 102 + # Attach an existing volume to instance. The volume will be deleted upon instance termination. - amazon.aws.ec2_vol: instance: XXXXXX @@ -218,13 +233,13 @@ returned: when success type: str sample: { - "attachment_set": { + "attachment_set": [{ "attach_time": "2015-10-23T00:22:29.000Z", "deleteOnTermination": "false", "device": "/dev/sdf", "instance_id": "i-8356263c", "status": "attached" - }, + }], "create_time": "2015-10-21T14:36:08.870Z", "encrypted": false, "id": "vol-35b333d9", @@ -408,7 +423,15 @@ def update_volume(module, ec2_conn, volume): throughput_changed = True req_obj['Throughput'] = target_throughput - changed = iops_changed or size_changed or type_changed or throughput_changed + target_multi_attach = module.params.get('multi_attach') + multi_attach_changed = False + if target_multi_attach is not None: + original_multi_attach = volume['multi_attach_enabled'] + if target_multi_attach != original_multi_attach: + multi_attach_changed = True + req_obj['MultiAttachEnabled'] = target_multi_attach + + changed = iops_changed or size_changed or type_changed or throughput_changed or multi_attach_changed if changed: response = ec2_conn.modify_volume(**req_obj) @@ -416,6 +439,7 @@ def update_volume(module, ec2_conn, volume): volume['size'] = response.get('VolumeModification').get('TargetSize') volume['volume_type'] = response.get('VolumeModification').get('TargetVolumeType') volume['iops'] = response.get('VolumeModification').get('TargetIops') + volume['multi_attach_enabled'] = response.get('VolumeModification').get('TargetMultiAttachEnabled') if module.botocore_at_least("1.19.27"): volume['throughput'] = response.get('VolumeModification').get('TargetThroughput') @@ -431,6 +455,7 @@ def create_volume(module, ec2_conn, zone): volume_type = module.params.get('volume_type') snapshot = module.params.get('snapshot') throughput = module.params.get('throughput') + multi_attach = module.params.get('multi_attach') volume = get_volume(module, ec2_conn) @@ -458,6 +483,8 @@ def create_volume(module, ec2_conn, zone): if throughput: additional_params['Throughput'] = int(throughput) + if multi_attach: + additional_params['MultiAttachEnabled'] = True create_vol_response = ec2_conn.create_volume( aws_retry=True, @@ -489,11 +516,13 @@ def attach_volume(module, ec2_conn, volume_dict, instance_dict, device_name): attachment_data = get_attachment_data(volume_dict, wanted_state='attached') if attachment_data: - if attachment_data.get('instance_id', None) != instance_dict['instance_id']: - module.fail_json(msg="Volume {0} is already attached to another instance: {1}".format(volume_dict['volume_id'], - attachment_data.get('instance_id', None))) - else: - return volume_dict, changed + if not volume_dict['multi_attach_enabled']: + # volumes without MultiAttach Enabled can be attached to 1 instance only + if attachment_data[0].get('instance_id', None) != instance_dict['instance_id']: + module.fail_json(msg="Volume {0} is already attached to another instance: {1}".format(volume_dict['volume_id'], + attachment_data[0].get('instance_id', None))) + else: + return volume_dict, changed try: attach_response = ec2_conn.attach_volume(aws_retry=True, Device=device_name, @@ -557,17 +586,22 @@ def modify_dot_attribute(module, ec2_conn, instance_dict, device_name): def get_attachment_data(volume_dict, wanted_state=None): changed = False - attachment_data = {} + attachment_data = [] if not volume_dict: return attachment_data - for data in volume_dict.get('attachments', []): - if wanted_state and wanted_state == data['state']: - attachment_data = data - break - else: - # No filter, return first - attachment_data = data - break + resource = volume_dict.get('attachments', []) + if wanted_state: + # filter 'state', return attachment matching wanted state + resource = [data for data in resource if data['state'] == wanted_state] + + for data in resource: + attachment_data.append({ + 'attach_time': data.get('attach_time', None), + 'device': data.get('device', None), + 'instance_id': data.get('instance_id', None), + 'status': data.get('state', None), + 'delete_on_termination': data.get('delete_on_termination', None) + }) return attachment_data @@ -576,8 +610,9 @@ def detach_volume(module, ec2_conn, volume_dict): changed = False attachment_data = get_attachment_data(volume_dict, wanted_state='attached') - if attachment_data: - ec2_conn.detach_volume(aws_retry=True, VolumeId=volume_dict['volume_id']) + # The ID of the instance must be specified if you are detaching a Multi-Attach enabled volume. + for attachment in attachment_data: + ec2_conn.detach_volume(aws_retry=True, InstanceId=attachment['instance_id'], VolumeId=volume_dict['volume_id']) waiter = ec2_conn.get_waiter('volume_available') waiter.wait( VolumeIds=[volume_dict['volume_id']], @@ -602,13 +637,8 @@ def get_volume_info(module, volume, tags=None): 'status': volume.get('state'), 'type': volume.get('volume_type'), 'zone': volume.get('availability_zone'), - 'attachment_set': { - 'attach_time': attachment_data.get('attach_time', None), - 'device': attachment_data.get('device', None), - 'instance_id': attachment_data.get('instance_id', None), - 'status': attachment_data.get('state', None), - 'deleteOnTermination': attachment_data.get('delete_on_termination', None) - }, + 'attachment_set': attachment_data, + 'multi_attach_enabled': volume.get('multi_attach_enabled'), 'tags': tags } @@ -659,6 +689,7 @@ def main(): modify_volume=dict(default=False, type='bool'), throughput=dict(type='int'), purge_tags=dict(type='bool', default=False), + multi_attach=dict(type='bool'), ) module = AnsibleAWSModule( @@ -681,6 +712,7 @@ def main(): iops = module.params.get('iops') volume_type = module.params.get('volume_type') throughput = module.params.get('throughput') + multi_attach = module.params.get('multi_attach') if state == 'list': module.deprecate( @@ -717,6 +749,9 @@ def main(): if throughput < 125 or throughput > 1000: module.fail_json(msg='Throughput values must be between 125 and 1000.') + if multi_attach is True and volume_type not in ('io1', 'io2'): + module.fail_json(msg='multi_attach is only supported for io1 and io2 volumes.') + # Set changed flag changed = False @@ -777,8 +812,6 @@ def main(): changed=False ) - attach_state_changed = False - if volume: volume, changed = update_volume(module, ec2_conn, volume) else: @@ -799,7 +832,7 @@ def main(): if tags_changed: changed = True - module.exit_json(changed=changed, volume=volume_info, device=volume_info['attachment_set']['device'], + module.exit_json(changed=changed, volume=volume_info, device=device_name, volume_id=volume_info['id'], volume_type=volume_info['type']) elif state == 'absent': if not name and not param_id: diff --git a/plugins/modules/ec2_vol_info.py b/plugins/modules/ec2_vol_info.py index ba20d45ee4f..45238ff9c70 100644 --- a/plugins/modules/ec2_vol_info.py +++ b/plugins/modules/ec2_vol_info.py @@ -109,6 +109,10 @@ description: The Availability Zone of the volume. type: str sample: "us-east-1b" + throughput: + description: The throughput that the volume supports, in MiB/s. + type: int + sample: 131 ''' try: @@ -128,6 +132,16 @@ def get_volume_info(volume, region): attachment = volume["attachments"] + attachment_data = [] + for data in volume["attachments"]: + attachment_data.append({ + 'attach_time': data.get('attach_time', None), + 'device': data.get('device', None), + 'instance_id': data.get('instance_id', None), + 'status': data.get('state', None), + 'delete_on_termination': data.get('delete_on_termination', None) + }) + volume_info = { 'create_time': volume["create_time"], 'id': volume["volume_id"], @@ -139,16 +153,13 @@ def get_volume_info(volume, region): 'type': volume["volume_type"], 'zone': volume["availability_zone"], 'region': region, - 'attachment_set': { - 'attach_time': attachment[0]["attach_time"] if len(attachment) > 0 else None, - 'device': attachment[0]["device"] if len(attachment) > 0 else None, - 'instance_id': attachment[0]["instance_id"] if len(attachment) > 0 else None, - 'status': attachment[0]["state"] if len(attachment) > 0 else None, - 'delete_on_termination': attachment[0]["delete_on_termination"] if len(attachment) > 0 else None - }, + 'attachment_set': attachment_data, 'tags': boto3_tag_list_to_ansible_dict(volume['tags']) if "tags" in volume else None } + if 'throughput' in volume: + volume_info['throughput'] = volume["throughput"] + return volume_info diff --git a/tests/integration/targets/ec2_vol/tasks/tests.yml b/tests/integration/targets/ec2_vol/tasks/tests.yml index d01b8942968..652ddb162a8 100644 --- a/tests/integration/targets/ec2_vol/tasks/tests.yml +++ b/tests/integration/targets/ec2_vol/tasks/tests.yml @@ -52,7 +52,7 @@ set_fact: ec2_ami_image: '{{ latest_ami.image_id }}' - # ==== ec2_vol tests =============================================== + # # ==== ec2_vol tests =============================================== - name: create a volume (validate module defaults) ec2_vol: @@ -73,8 +73,7 @@ - volume1.volume.status == 'available' - volume1.volume_type == 'standard' - "'attachment_set' in volume1.volume" - - "'instance_id' in volume1.volume.attachment_set" - - not volume1.volume.attachment_set.instance_id + - volume1.volume.attachment_set | length == 0 - not ("Name" in volume1.volume.tags) - not volume1.volume.encrypted - volume1.volume.tags.ResourcePrefix == "{{ resource_prefix }}" @@ -185,12 +184,10 @@ - vol_attach_result.changed - "'device' in vol_attach_result and vol_attach_result.device == '/dev/sdg'" - "'volume' in vol_attach_result" - - vol_attach_result.volume.attachment_set.status in ['attached', 'attaching'] - - vol_attach_result.volume.attachment_set.instance_id == test_instance.instance_ids[0] - - vol_attach_result.volume.attachment_set.device == '/dev/sdg' - -# Failing -# - "vol_attach_result.volume.attachment_set.deleteOnTermination" + - vol_attach_result.volume.attachment_set[0].status in ['attached', 'attaching'] + - vol_attach_result.volume.attachment_set[0].instance_id == test_instance.instance_ids[0] + - vol_attach_result.volume.attachment_set[0].device == '/dev/sdg' + - not vol_attach_result.volume.attachment_set[0].delete_on_termination - name: attach existing volume to an instance (idempotent) ec2_vol: @@ -204,7 +201,7 @@ assert: that: - "not vol_attach_result.changed" - - vol_attach_result.volume.attachment_set.status in ['attached', 'attaching'] + - vol_attach_result.volume.attachment_set[0].status in ['attached', 'attaching'] - name: attach a new volume to an instance ec2_vol: @@ -227,9 +224,9 @@ - new_vol_attach_result.changed - "'device' in new_vol_attach_result and new_vol_attach_result.device == '/dev/sdh'" - "'volume' in new_vol_attach_result" - - new_vol_attach_result.volume.attachment_set.status in ['attached', 'attaching'] - - new_vol_attach_result.volume.attachment_set.instance_id == test_instance.instance_ids[0] - - new_vol_attach_result.volume.attachment_set.device == '/dev/sdh' + - new_vol_attach_result.volume.attachment_set[0].status in ['attached', 'attaching'] + - new_vol_attach_result.volume.attachment_set[0].instance_id == test_instance.instance_ids[0] + - new_vol_attach_result.volume.attachment_set[0].device == '/dev/sdh' - new_vol_attach_result.volume.tags["lowercase spaced"] == 'hello cruel world' - new_vol_attach_result.volume.tags["Title Case"] == 'Hello Cruel World' - new_vol_attach_result.volume.tags["CamelCase"] == 'SimpleCamelCase' @@ -306,8 +303,8 @@ - attach_new_vol_from_snapshot_result.changed - "'device' in attach_new_vol_from_snapshot_result and attach_new_vol_from_snapshot_result.device == '/dev/sdi'" - "'volume' in attach_new_vol_from_snapshot_result" - - attach_new_vol_from_snapshot_result.volume.attachment_set.status in ['attached', 'attaching'] - - attach_new_vol_from_snapshot_result.volume.attachment_set.instance_id == test_instance.instance_ids[0] + - attach_new_vol_from_snapshot_result.volume.attachment_set[0].status in ['attached', 'attaching'] + - attach_new_vol_from_snapshot_result.volume.attachment_set[0].instance_id == test_instance.instance_ids[0] - name: list volumes attached to instance ec2_vol: @@ -454,7 +451,7 @@ - name: volume type must be gp3 assert: that: - - v.type == 'gp3' + - v.type == 'gp3' vars: v: "{{ verify_gp3_change.volumes[0] }}" @@ -501,8 +498,7 @@ that: - dot_volume.changed - "'attachment_set' in dot_volume.volume" - - "'deleteOnTermination' in dot_volume.volume.attachment_set" - - "dot_volume.volume.attachment_set.deleteOnTermination is defined" + - "'delete_on_termination' in dot_volume.volume.attachment_set[0]" - "'create_time' in dot_volume.volume" - "'id' in dot_volume.volume" - "'size' in dot_volume.volume" @@ -537,10 +533,10 @@ assert: that: - "volume_info.volumes|length == 1" - - "v.attachment_set.attach_time is defined" - - "v.attachment_set.device is defined and v.attachment_set.device == dot_volume.device" - - "v.attachment_set.instance_id is defined and v.attachment_set.instance_id == test_instance.instance_ids[0]" - - "v.attachment_set.status is defined and v.attachment_set.status == 'attached'" + - "v.attachment_set[0].attach_time is defined" + - "v.attachment_set[0].device is defined and v.attachment_set[0].device == dot_volume.device" + - "v.attachment_set[0].instance_id is defined and v.attachment_set[0].instance_id == test_instance.instance_ids[0]" + - "v.attachment_set[0].status is defined and v.attachment_set[0].status == 'attached'" - "v.create_time is defined" - "v.encrypted is defined and v.encrypted == false" - "v.id is defined and v.id == dot_volume.volume_id" @@ -559,14 +555,14 @@ - name: New format check assert: that: - - "v.attachment_set.delete_on_termination is defined" + - "v.attachment_set[0].delete_on_termination is defined" vars: v: "{{ volume_info.volumes[0] }}" when: ansible_version.full is version('2.7', '>=') - name: test create a new gp3 volume ec2_vol: - volume_size: 7 + volume_size: 70 zone: "{{ availability_zone }}" volume_type: gp3 throughput: 130 @@ -581,12 +577,10 @@ that: - gp3_volume.changed - "'attachment_set' in gp3_volume.volume" - - "'deleteOnTermination' in gp3_volume.volume.attachment_set" - - gp3_volume.volume.attachment_set.deleteOnTermination == none - "'create_time' in gp3_volume.volume" - "'id' in gp3_volume.volume" - "'size' in gp3_volume.volume" - - gp3_volume.volume.size == 7 + - gp3_volume.volume.size == 70 - "'volume_type' in gp3_volume" - gp3_volume.volume_type == 'gp3' - "'iops' in gp3_volume.volume" @@ -597,9 +591,43 @@ - (gp3_volume.volume.tags | length ) == 2 - gp3_volume.volume.tags["ResourcePrefix"] == "{{ resource_prefix }}" + - name: Read volume information to validate throughput + ec2_vol_info: + filters: + volume-id: "{{ gp3_volume.volume_id }}" + register: verify_throughput + + - name: throughput must be equal to 130 + assert: + that: + - v.throughput == 130 + vars: + v: "{{ verify_throughput.volumes[0] }}" + + - name: print out facts + debug: + var: vol_facts + + - name: Read volume information to validate throughput + ec2_vol_info: + filters: + volume-id: "{{ gp3_volume.volume_id }}" + register: verify_throughput + + - name: throughput must be equal to 130 + assert: + that: + - v.throughput == 130 + vars: + v: "{{ verify_throughput.volumes[0] }}" + + - name: print out facts + debug: + var: vol_facts + - name: increase throughput ec2_vol: - volume_size: 7 + volume_size: 70 zone: "{{ availability_zone }}" volume_type: gp3 throughput: 131 @@ -613,23 +641,75 @@ assert: that: - gp3_volume.changed - - "'attachment_set' in gp3_volume.volume" - - "'deleteOnTermination' in gp3_volume.volume.attachment_set" - - gp3_volume.volume.attachment_set.deleteOnTermination == none - "'create_time' in gp3_volume.volume" - "'id' in gp3_volume.volume" - "'size' in gp3_volume.volume" - - gp3_volume.volume.size == 7 + - gp3_volume.volume.size == 70 - "'volume_type' in gp3_volume" - gp3_volume.volume_type == 'gp3' - "'iops' in gp3_volume.volume" - gp3_volume.volume.iops == 3001 - "'throughput' in gp3_volume.volume" - gp3_volume.volume.throughput == 131 - - "'tags' in gp3_volume.volume" - - (gp3_volume.volume.tags | length ) == 2 - - gp3_volume.volume.tags["ResourcePrefix"] == "{{ resource_prefix }}" + + # Multi-Attach disk + - name: create disk with multi-attach enabled + ec2_vol: + volume_size: 4 + volume_type: io1 + iops: 102 + zone: "{{ availability_zone }}" + multi_attach: yes + tags: + ResourcePrefix: "{{ resource_prefix }}" + register: multi_attach_disk + + - name: check volume creation + assert: + that: + - multi_attach_disk.changed + - "'volume' in multi_attach_disk" + - multi_attach_disk.volume.multi_attach_enabled + + - name: attach existing volume to an instance + ec2_vol: + id: "{{ multi_attach_disk.volume_id }}" + instance: "{{ test_instance.instance_ids[0] }}" + device_name: /dev/sdk + delete_on_termination: no + register: vol_attach_result + + - name: create another ec2 instance + ec2_instance: + name: "{{ resource_prefix }}-2" + vpc_subnet_id: "{{ testing_subnet.subnet.id }}" + instance_type: t3.nano + image_id: "{{ ec2_ami_image }}" + tags: + ResourcePrefix: "{{ resource_prefix }}" + register: test_instance_2 + + - name: check task return attributes + assert: + that: + - test_instance_2.changed + + - name: attach existing volume to second instance + ec2_vol: + id: "{{ multi_attach_disk.volume_id }}" + instance: "{{ test_instance_2.instance_ids[0] }}" + device_name: /dev/sdg + delete_on_termination: no + register: vol_attach_result + - name: check task return attributes + assert: + that: + - vol_attach_result.changed + - "'volume' in vol_attach_result" + - vol_attach_result.volume.attachment_set | length == 2 + - 'test_instance.instance_ids[0] in vol_attach_result.volume.attachment_set | map(attribute="instance_id") | list' + - 'test_instance_2.instance_ids[0] in vol_attach_result.volume.attachment_set | map(attribute="instance_id") | list' # ==== Cleanup ============================================================ @@ -637,8 +717,11 @@ - name: Describe the instance before we delete it ec2_instance_info: instance_ids: - - "{{ test_instance.instance_ids[0] }}" + - "{{ item }}" ignore_errors: yes + with_items: + - "{{ test_instance.instance_ids[0] }}" + - "{{ test_instance_2.instance_ids[0] }}" register: pre_delete - debug: @@ -647,8 +730,11 @@ - name: delete test instance ec2_instance: instance_ids: - - "{{ test_instance.instance_ids[0] }}" + - "{{ item }}" state: terminated + with_items: + - "{{ test_instance.instance_ids[0] }}" + - "{{ test_instance_2.instance_ids[0] }}" ignore_errors: yes - name: delete volumes @@ -664,6 +750,7 @@ - "{{ attach_new_vol_from_snapshot_result }}" - "{{ dot_volume }}" - "{{ gp3_volume }}" + - "{{ multi_attach_disk }}" - name: delete snapshot ec2_snapshot: