Restore nodes from individual EBS snapshots

_dependencies/tasks/main.yml

@@ -4,14 +4,14 @@
   include_vars: { dir: "{{ playbook_dir }}/group_vars/{{ clusterid }}" }
   tags: clusterverse_clean,clusterverse_create,clusterverse_dynamic_inventory,clusterverse_config,clusterverse_readiness
 
-- name: Preflight check
-  block:
-    - assert: { that: "ansible_version.full is version_compare('2.9', '>=')", msg: "Ansible >=2.9 required." }
-    - assert: { that: "app_name is defined and app_name != ''", msg: "Please define app_name" }
-    - assert: { that: "app_class is defined and app_class != ''", msg: "Please define app_class" }
-    - assert: { that: "clusterid is defined and cluster_vars is defined", msg: "Please define clusterid" }
-    - assert: { that: "buildenv is defined and cluster_vars[buildenv] is defined", msg: "Please define buildenv" }
+# - name: Preflight check
+#   block:
+#     - assert: { that: "ansible_version.full is version_compare('2.9', '>=')", msg: "Ansible >=2.9 required." }
+#     - assert: { that: "app_name is defined and app_name != ''", msg: "Please define app_name" }
+#     - assert: { that: "app_class is defined and app_class != ''", msg: "Please define app_class" }
+#     - assert: { that: "clusterid is defined and cluster_vars is defined", msg: "Please define clusterid" }
+#     - assert: { that: "buildenv is defined and cluster_vars[buildenv] is defined", msg: "Please define buildenv" }
 
-    - assert: { that: "(cluster_vars.assign_public_ip == 'yes' and cluster_vars.inventory_ip == 'public') or (cluster_vars.inventory_ip == 'private')", msg: "If inventory_ip=='public', 'assign_public_ip' must be 'yes'" }
-      when: cluster_vars.type == "gce" or cluster_vars.type == "aws"
-  tags: clusterverse_clean,clusterverse_create,clusterverse_dynamic_inventory,clusterverse_config,clusterverse_readiness
+#     - assert: { that: "(cluster_vars.assign_public_ip == 'yes' and cluster_vars.inventory_ip == 'public') or (cluster_vars.inventory_ip == 'private')", msg: "If inventory_ip=='public', 'assign_public_ip' must be 'yes'" }
+#       when: cluster_vars.type == "gce" or cluster_vars.type == "aws"
+#   tags: clusterverse_clean,clusterverse_create,clusterverse_dynamic_inventory,clusterverse_config,clusterverse_readiness

cluster_hosts/tasks/main.yml

@@ -4,29 +4,37 @@
   set_fact:
     epoch_time: "{{ansible_date_time.epoch}}"
 
-# Create an array of dictionaries containing all the hostnames PER-AZ (i.e. couchbase-dev-node-a0, couchbase-dev-master-a1, couchbase-dev-master-b0, couchbase-dev-master-b1 etc) to be created:
 - set_fact:
     cluster_hosts_flat: |
       {% set res = [] -%}
       {%- for hostttype in cluster_vars[buildenv].hosttype_vars.keys() -%}
+        {%- set _count = [] %}
         {%- for azname in cluster_vars[buildenv].hosttype_vars[hostttype].vms_by_az.keys() -%}
           {%- for azcount in range(0,cluster_vars[buildenv].hosttype_vars[hostttype].vms_by_az[azname]|int) -%}
+            {%- set _c_dummy = _count.append(1) -%}
             {% set _dummy = res.extend([{
               'hosttype': hostttype,
               'hostname': cluster_name + '-' + hostttype + '-' + azname + azcount|string,
               'az_name': azname|string,
               'flavor': cluster_vars[buildenv].hosttype_vars[hostttype].flavor,
-              'release': release_version
+              'release': release_version,
+              'auto_volumes': cluster_vars[buildenv].hosttype_vars[hostttype].auto_volumes,
+              'index_per_hosttype': (_count|length - 1)
               }]) -%}
           {%- endfor %}
         {%- endfor %}
       {%- endfor %}
       {{ res }}
 
+
+
 - include_tasks: aws.yml
   when: cluster_vars.type == "aws"
 
 - include_tasks: gce.yml
   when: cluster_vars.type == "gce"
 
 #- debug: msg={{cluster_hosts_flat}}
+
+
+# Correcting @eyettea 's clean solution: Another solution that's a bit cleaner imo is to initialize an empty list {% set count = [ ] %}, add an item to the list in every loop {% set __ = count.append(1) %} and use the length to display the count count|length 

create/tasks/aws.yml

@@ -1,4 +1,42 @@
 ---
+- block:
+  - name: Get snapshots info
+    ec2_snapshot_info:
+      aws_access_key: "{{ lookup('env','AWS_ACCESS_KEY_ID') }}"
+      aws_secret_key: "{{ lookup('env','AWS_SECRET_ACCESS_KEY') }}"
+      region: "{{ cluster_vars.region }}"
+      filters: "{{ _snapshot_tags[0] }}"
+    register: r_ebs_snapshots
+
+  - name: Assert that snapshots exists
+    assert:
+      that:
+        - item in _available_snapshots
+      quiet: true
+      fail_msg: "{{ item }}  not in available snapshots {{ _available_snapshots }}"
+    loop: "{{ _configuration_snapshots }}"
+    vars:
+      _available_snapshots: "{{ r_ebs_snapshots.snapshots|json_query('[].tags.backup_id')|unique }}"
+      _configuration_snapshots: "{{ cluster_vars[buildenv].hosttype_vars|json_query('*.auto_volumes[].snapshot_tags.*[]')|unique }}"
+
+  - include_tasks: aws_disks.yml
+    loop_control:
+      loop_var: loop_instance
+    loop: "{{ cluster_hosts_flat|sort(attribute='hostname') }}"
+    vars:
+      _available_snapshots: "{{ r_ebs_snapshots.snapshots|json_query('[].tags.backup_id')|unique }}"
+
+  - set_fact:
+      cluster_hosts_flat: "{{ cluster_hosts_flat_snapshots }}"
+
+  vars:
+    _snapshot_tags: "{{ cluster_vars[buildenv].hosttype_vars|json_query('*.auto_volumes[].snapshot_tags') }}"
+    _snapshot_tags_check: "{{ _snapshot_tags[0] if _snapshot_tags|length > 0 else {} }}"
+  when:
+    - _snapshot_tags_check != {}
+
+- debug:
+    msg: "{{ cluster_hosts_flat }}"
 
 - name: Create AWS security group
   ec2_group:
@@ -43,7 +81,7 @@
           owner: "{{ lookup('env','USER')| lower }}"
           maintenance_mode: "{%- if prometheus_set_unset_maintenance_mode|bool -%}true{%- else -%}false{%- endif -%}"
         termination_protection: "{{cluster_vars[buildenv].termination_protection}}"
-        volumes: "{{cluster_vars[buildenv].hosttype_vars[item.hosttype].auto_volumes | default([])}}"
+        volumes: "{{item.auto_volumes | default([])}}"
         count_tag:
           Name: "{{item.hostname}}"
         exact_count: 1

create/tasks/aws_disks.yml

@@ -0,0 +1,16 @@
+- set_fact:
+    auto_volumes_snapshots: "{{ auto_volumes_snapshots|default([]) + [_auto_instance] }}"
+  loop: "{{ loop_instance.auto_volumes }}"
+  vars:
+    _auto_instance: "{% if 'snapshot_tags' in item.keys() %}{{ item|combine({'snapshot': _snapshot_id }) }}{% else %}{{ item }}{% endif %}"
+    _snapshot_id: "{% if 'snapshot_tags' in item.keys() %}{{ (r_ebs_snapshots.snapshots|json_query(_query))[0].snapshot_id }}{% endif %}"
+    _query: "{% if 'snapshot_tags' in item.keys() %}[?tags.backup_id == '{{ item.snapshot_tags['tag:backup_id']}}' && tags.node == '{{ loop_instance.index_per_hosttype }}']{% endif %}"
+
+- set_fact:
+    instance_with_auto_snapshots: "{{ instance_with_auto_snapshots|default({})|combine({'auto_volumes': auto_volumes_snapshots }) }}"
+
+- set_fact:
+    cluster_hosts_flat_snapshots: "{{ cluster_hosts_flat_snapshots|default([]) + [ _cluster_hosts_flat_snapshots_loop ]}}"
+    auto_volumes_snapshots: []
+  vars:
+    _cluster_hosts_flat_snapshots_loop: "{{ loop_instance| combine(instance_with_auto_snapshots) }}"