Revert route53_info back to route53(state=get) with valid Ansible ver…

…sions.

+ Using route53 info in clean/tasks/dns.yml is too slow with large recordsets, (hopefully only until ansible-collections/community.aws#529 is fixed).
+ It is only supported between Ansible 2.9.6 and 2.10.6 and after Ansible 4; assertions added to pre-flight check.

EXAMPLE/cluster_defs/esxifree/testid/homelab/cluster_vars__region.yml

@@ -1,7 +1,7 @@
 ---
 
-_ubuntu2004image: "gold-ubuntu2004l-20210415101808"
-_centos7image: "gold-ubuntu2004l-20210415101808"
+_ubuntu2004image: "gold-ubuntu2004l-20210610082838"
+_centos7image: "gold-ubuntu2004l-20210610082838"
 
 cluster_vars:
   image: "{{_ubuntu2004image}}"

EXAMPLE/cluster_defs/esxifree/testid/homelab/sandbox/cluster_vars__buildenv.yml

@@ -21,7 +21,7 @@ cluster_vars:
         flavor: { num_cpus: "2", memory_mb: "2048" }
         version: "{{sys_version | default('')}}"
         networks: [ &sys_NET1 { networkName: "VM Network", virtualDev: vmxnet3 } ]
-        vms_by_az: { a: 2, b: 1, c: 0 }
+        vms_by_az: { a: 1, b: 1, c: 0 }
 
       sysdisks2:
         auto_volumes:

EXAMPLE/jenkinsfiles/Jenkinsfile_ops

@@ -2,7 +2,7 @@
 
 //These will not be needed if we're running this as a pipeline SCM job, as these are automatically added to the 'scm' variable, but if we instead just cut & paste this file into a pipeline job, they will be used as fallback
 def DEFAULT_CLUSTERVERSE_URL = "https://github.com/dseeley/clusterverse"
-def DEFAULT_CLUSTERVERSE_BRANCH = "dps_esxi"
+def DEFAULT_CLUSTERVERSE_BRANCH = "master"
 
 def DEFAULT_CLUSTERVERSE_TESTSUITE_URL = "https://github.com/dseeley/clusterverse_test"
 def DEFAULT_CLUSTERVERSE_TESTSUITE_BRANCH = "master"

README.md

@@ -1,6 +1,6 @@
 # clusterverse    [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) ![PRs Welcome](https://img.shields.io/badge/PRs-Welcome-brightgreen.svg)
 A full-lifecycle, immutable cloud infrastructure cluster management **role**, using Ansible.
-+ **Multi-cloud:** clusterverse can manage cluster lifecycle in AWS, GCP, Free ESXi (standalone host only, not vCentre) and Azure
++ **Multi-cloud:** clusterverse can manage cluster lifecycle in AWS, GCP, Azure and Free ESXi (standalone host only, not vCentre).
 + **Deploy:**  You define your infrastructure as code (in Ansible yaml), and clusterverse will deploy it 
 + **Scale-up:**  If you change the cluster definitions and rerun the deploy, new nodes will be added.
 + **Redeploy (e.g. up-version):** If you need to up-version, or replace the underlying OS, (i.e. to achieve fully immutable, zero-patching redeploys), the `redeploy.yml` playbook will replace each node in the cluster (via various redeploy schemes), and rollback if any failures occur. 

_dependencies/tasks/main.yml

@@ -30,7 +30,14 @@
 
 - name: Preflight check
   block:
-    - assert: { that: "ansible_version.full is version_compare('2.10', '>=')", fail_msg: "Ansible > 2.10 required for Azure support." }
+    - assert: { that: "ansible_version.full is version_compare('2.10', '>=')", fail_msg: "ansible-core > 2.10 required for Azure support." }
+      when: cluster_vars.type == "azure"
+
+    - assert:
+        that: "(ansible_version.full is version_compare('2.9.6', '>=') and ansible_version.full is version_compare('2.10.6', '<='))  or  galaxy_collections['community.aws'].version is version_compare('1.5.0', '>=')"
+        fail_msg: "If Ansible > 2.9.6 then community.aws > 1.5.0 is required for valid community.aws.route53 support (by default in Ansible v4)."
+      vars: { galaxy_collections: "{{ (lookup('pipe', 'ansible-galaxy collection list --format json')) | from_json |  json_query(\"* | [0]\") }}" }
+
     - assert: { that: "app_name is defined and app_name != ''", fail_msg: "Please define app_name" }
     - assert: { that: "app_class is defined and app_class != ''", fail_msg: "Please define app_class" }
     - assert: { that: "cluster_vars is defined", fail_msg: "Please define cluster_vars" }

clean/tasks/dns.yml

@@ -35,81 +35,71 @@
 
     - name: clean/dns/route53 | Delete DNS entries
       block:
-        - name: clean/dns/route53 | Get Zone
-          route53_zone:
-            aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}"
-            aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}"
-            zone: "{{cluster_vars.dns_nameserver_zone}}"
-            vpc_id: "{{ vpc_id if cluster_vars.route53_private_zone|bool else omit }}"                     # If this is a private zone, we need to request the private (VPC/Region-specific) version of the Zone
-            vpc_region: "{{ cluster_vars.region if cluster_vars.route53_private_zone|bool else omit }}"    # If this is a private zone, we need to request the private (VPC/Region-specific) version of the Zone
-          register: r__route53_zone
-
-        # Note: route53_info currently does not honour the 'max_items' or 'type' fields, (and if 'start_record_name' is not found, it just returns all records), so we need to filter the responses to match 'hosts_to_clean' when doing the delete
-        # Note: cannot run route53_info asynchronously as it makes too many concurrent requests and blows the AWS Route53 API limit.
         - name: clean/dns/route53 | Get A records
-          route53_info:
+          route53:
             aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}"
             aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}"
+            state: "get"
+            zone: "{{cluster_vars.dns_nameserver_zone}}"
+            record: "{{item.name}}.{{cluster_vars.dns_user_domain}}"
             type: "A"
-            max_items: 1
-            query: record_sets
-            hosted_zone_id: "{{ r__route53_zone.zone_id }}"
-            start_record_name: "{{item.name}}.{{cluster_vars.dns_user_domain}}"
-          register: r__route53_info
-          until: r__route53_info is success
-          retries: 10
+            private_zone: "{{cluster_vars.route53_private_zone | default(true)}}"
+          register: r__route53_a
           with_items: "{{ hosts_to_clean }}"
+          ignore_errors: yes
+        - name: clean/dns/route53 | Remove failed DNS lookups from route53 state=get
+          set_fact:
+            r__route53_a: "{{r__route53_a.results | selectattr('failed', '==', false) | list}}"
+          when: r__route53_a is failed
+
+        - debug: msg={{r__route53_a}}
 
         - name: clean/dns/route53 | Delete A records
           route53:
             aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}"
             aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}"
             state: "absent"
-            zone: "{{ cluster_vars.dns_nameserver_zone }}"
-            record: "{{ item.Name }}"
-            type: "{{ item.Type }}"
-            ttl: "{{ item.TTL }}"
-            value: "{{ item.ResourceRecords | json_query(\"[].Value\") }}"
+            zone: "{{ item.set.zone }}"
+            record: "{{ item.set.record }}"
+            type: "{{ item.set.type }}"
+            ttl: "{{ item.set.ttl }}"
+            value: ["{{ item.set.value }}"]
             private_zone: "{{cluster_vars.route53_private_zone | default(true)}}"
-          with_items: "{{ records_to_clean }}"
-          vars:
-            _hostnames_to_clean: "{{ hosts_to_clean | json_query(\"[].name\") | map('regex_replace', '^(.*)$', '\\1.' + cluster_vars.dns_user_domain + '.') | list }}"
-            records_to_clean: "{{ r__route53_info.results | json_query(\"[].ResourceRecordSets[?Type=='A' && contains(\"+ _hostnames_to_clean | string +\", Name)][]\") | unique }}"
+          with_items: "{{ r__route53_a.results }}"
+          when: item.set.value is defined
 
-        # Note: route53_info currently does not honour the 'max_items' or 'type' fields, (and if 'start_record_name' is not found, it just returns all records), so we need to filter the responses to match 'hosts_to_clean' when doing the delete
-        # Note: cannot run route53_info asynchronously as it makes too many concurrent requests and blows the AWS Route53 API limit.
         - name: clean/dns/route53 | Get CNAME records
-          route53_info:
+          route53:
             aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}"
             aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}"
+            state: "get"
+            zone: "{{cluster_vars.dns_nameserver_zone}}"
+            record: "{{item.name | regex_replace('-(?!.*-).*')}}.{{cluster_vars.dns_user_domain}}"
             type: "CNAME"
-            max_items: 1
-            query: record_sets
-            hosted_zone_id: "{{ r__route53_zone.zone_id }}"
-            start_record_name: "{{item.name | regex_replace('-(?!.*-).*')}}.{{cluster_vars.dns_user_domain}}"
-          register: r__route53_info
+            private_zone: "{{cluster_vars.route53_private_zone | default(true)}}"
+          register: r__route53_cname
           with_items: "{{ hosts_to_clean }}"
-          until: r__route53_info is success
-          retries: 10
+          ignore_errors: yes
+        - name: clean/dns/route53 | Remove failed DNS lookups from route53 state=get
+          set_fact:
+            r__route53_cname: "{{r__route53_cname.results | selectattr('failed', '==', false) | list}}"
+          when: r__route53_cname is failed
 
         - name: clean/dns/route53 | Delete CNAME records
           route53:
             aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}"
             aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}"
             state: "absent"
-            zone: "{{ cluster_vars.dns_nameserver_zone }}"
-            record: "{{ item.1.Name }}"
-            type: "{{ item.1.Type }}"
-            ttl: "{{ item.1.TTL }}"
-            value: "{{ item.1.ResourceRecords | json_query(\"[].Value\") }}"
+            zone: "{{ item.1.set.zone }}"
+            record: "{{ item.1.set.record }}"
+            type: "{{ item.1.set.type }}"
+            ttl: "{{ item.1.set.ttl }}"
+            value: ["{{ item.1.set.value }}"]
             private_zone: "{{cluster_vars.route53_private_zone | default(true)}}"
-          vars:
-            _cnames_to_clean: "{{ hosts_to_clean | json_query(\"[].name\") | map('regex_replace', '^(.*)-(?!.*-).*$', '\\1.' + cluster_vars.dns_user_domain + '.') | list }}"   #Remove the last '-.*' (cluster_suffix)
-            records_to_clean: "{{ r__route53_info.results | json_query(\"[].ResourceRecordSets[?Type=='CNAME' && contains(\"+ _cnames_to_clean | string +\", Name)][]\") | unique }}"
           with_nested:
             - "{{ hosts_to_clean }}"
-            - "{{ records_to_clean }}"
-          when: ((item.0.name | regex_replace('-(?!.*-).*')) == (item.1.Name | regex_replace('^(.*?)\\..*$', '\\1')))  and  ((item.0.name == item.1.ResourceRecords[0].Value | regex_replace('^(.*?)\\..*$', '\\1')))
+            - "{{ r__route53_cname.results }}"
+          when: (item.1.set.value is defined)  and  ((item.0.name | regex_replace('-(?!.*-).*')) == (item.1.set.record | regex_replace('^(.*?)\\..*$', '\\1')))  and  (item.0.name == item.1.set.value | regex_replace('^(.*?)\\..*$', '\\1'))
       when: cluster_vars.dns_server == "route53"
 
     - name: clean/dns/clouddns | Delete DNS entries