Revert route53_info back to route53(state=get) with valid Ansible ve…

…rsions. (#3)

+ Using `route53_zone`/ `route53_info`/ `route53` in clean/tasks/dns.yml is too slow with large recordsets, (hopefully only until ansible-collections/community.aws#529 is fixed).
+ Old mechanism (`route53(state=get)`) is only supported between Ansible 2.9.6 and 2.10.6 and after Ansible 4; assertions added to pre-flight check.

EXAMPLE/cluster_defs/esxifree/testid/homelab/cluster_vars__region.yml

@@ -1,7 +1,7 @@
 ---
 
-_ubuntu2004image: "gold-ubuntu2004l-20210415101808"
-_centos7image: "gold-ubuntu2004l-20210415101808"
+_ubuntu2004image: "gold-ubuntu2004l-20210610082838"
+_centos7image: "gold-ubuntu2004l-20210610082838"
 
 cluster_vars:
   image: "{{_ubuntu2004image}}"

EXAMPLE/cluster_defs/esxifree/testid/homelab/sandbox/cluster_vars__buildenv.yml

@@ -21,7 +21,7 @@ cluster_vars:
         flavor: { num_cpus: "2", memory_mb: "2048" }
         version: "{{sys_version | default('')}}"
         networks: [ &sys_NET1 { networkName: "VM Network", virtualDev: vmxnet3 } ]
-        vms_by_az: { a: 2, b: 1, c: 0 }
+        vms_by_az: { a: 1, b: 1, c: 0 }
 
       sysdisks2:
         auto_volumes:

EXAMPLE/jenkinsfiles/Jenkinsfile_ops

@@ -2,7 +2,7 @@
 
 //These will not be needed if we're running this as a pipeline SCM job, as these are automatically added to the 'scm' variable, but if we instead just cut & paste this file into a pipeline job, they will be used as fallback
 def DEFAULT_CLUSTERVERSE_URL = "https://github.com/dseeley/clusterverse"
-def DEFAULT_CLUSTERVERSE_BRANCH = "dps_esxi"
+def DEFAULT_CLUSTERVERSE_BRANCH = "master"
 
 def DEFAULT_CLUSTERVERSE_TESTSUITE_URL = "https://github.com/dseeley/clusterverse_test"
 def DEFAULT_CLUSTERVERSE_TESTSUITE_BRANCH = "master"

README.md

@@ -1,6 +1,6 @@
 # clusterverse    [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) ![PRs Welcome](https://img.shields.io/badge/PRs-Welcome-brightgreen.svg)
 A full-lifecycle, immutable cloud infrastructure cluster management **role**, using Ansible.
-+ **Multi-cloud:** clusterverse can manage cluster lifecycle in AWS, GCP, Free ESXi (standalone host only, not vCentre) and Azure
++ **Multi-cloud:** clusterverse can manage cluster lifecycle in AWS, GCP, Azure and Free ESXi (standalone host only, not vCentre).
 + **Deploy:**  You define your infrastructure as code (in Ansible yaml), and clusterverse will deploy it 
 + **Scale-up:**  If you change the cluster definitions and rerun the deploy, new nodes will be added.
 + **Redeploy (e.g. up-version):** If you need to up-version, or replace the underlying OS, (i.e. to achieve fully immutable, zero-patching redeploys), the `redeploy.yml` playbook will replace each node in the cluster (via various redeploy schemes), and rollback if any failures occur. 

_dependencies/tasks/main.yml

@@ -30,7 +30,14 @@
 
 - name: Preflight check
   block:
-    - assert: { that: "ansible_version.full is version_compare('2.10', '>=')", fail_msg: "Ansible > 2.10 required for Azure support." }
+    - assert: { that: "ansible_version.full is version_compare('2.10', '>=')", fail_msg: "ansible-core > 2.10 required for Azure support." }
+      when: cluster_vars.type == "azure"
+
+    - assert:
+        that: "(ansible_version.full is version_compare('2.9.6', '>=') and ansible_version.full is version_compare('2.10.6', '<='))  or  galaxy_collections['community.aws'].version is version_compare('1.5.0', '>=')"
+        fail_msg: "If Ansible > 2.9.6 then community.aws > 1.5.0 is required for valid community.aws.route53 support (by default in Ansible v4)."
+      vars: { galaxy_collections: "{{ (lookup('pipe', 'ansible-galaxy collection list --format json')) | from_json |  json_query(\"* | [0]\") }}" }
+
     - assert: { that: "app_name is defined and app_name != ''", fail_msg: "Please define app_name" }
     - assert: { that: "app_class is defined and app_class != ''", fail_msg: "Please define app_class" }
     - assert: { that: "cluster_vars is defined", fail_msg: "Please define cluster_vars" }

clean/tasks/dns.yml

@@ -33,7 +33,76 @@
           when: (item.name + '.' + cluster_vars.dns_user_domain + "." == cname_value)
       when: cluster_vars.dns_server == "nsupdate"
 
-    - name: clean/dns/route53 | Delete DNS entries
+
+    - name: "clean/dns/route53 | Delete DNS entries (legacy (faster) mechanism: route53(state=get)/ route53)"
+      block:
+        - name: clean/dns/route53 | Get A records
+          route53:
+            aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}"
+            aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}"
+            state: "get"
+            zone: "{{cluster_vars.dns_nameserver_zone}}"
+            record: "{{item.name}}.{{cluster_vars.dns_user_domain}}"
+            type: "A"
+            private_zone: "{{cluster_vars.route53_private_zone | default(true)}}"
+          register: r__route53_a
+          with_items: "{{ hosts_to_clean }}"
+          ignore_errors: yes
+        - name: clean/dns/route53 | Remove failed DNS lookups from route53 state=get
+          set_fact:
+            r__route53_a: "{{r__route53_a.results | selectattr('failed', '==', false) | list}}"
+          when: r__route53_a is failed
+
+        - name: clean/dns/route53 | Delete A records
+          route53:
+            aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}"
+            aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}"
+            state: "absent"
+            zone: "{{ item.set.zone }}"
+            record: "{{ item.set.record }}"
+            type: "{{ item.set.type }}"
+            ttl: "{{ item.set.ttl }}"
+            value: ["{{ item.set.value }}"]
+            private_zone: "{{cluster_vars.route53_private_zone | default(true)}}"
+          with_items: "{{ r__route53_a.results }}"
+          when: item.set.value is defined
+
+        - name: clean/dns/route53 | Get CNAME records
+          route53:
+            aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}"
+            aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}"
+            state: "get"
+            zone: "{{cluster_vars.dns_nameserver_zone}}"
+            record: "{{item.name | regex_replace('-(?!.*-).*')}}.{{cluster_vars.dns_user_domain}}"
+            type: "CNAME"
+            private_zone: "{{cluster_vars.route53_private_zone | default(true)}}"
+          register: r__route53_cname
+          with_items: "{{ hosts_to_clean }}"
+          ignore_errors: yes
+        - name: clean/dns/route53 | Remove failed DNS lookups from route53 state=get
+          set_fact:
+            r__route53_cname: "{{r__route53_cname.results | selectattr('failed', '==', false) | list}}"
+          when: r__route53_cname is failed
+
+        - name: clean/dns/route53 | Delete CNAME records
+          route53:
+            aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}"
+            aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}"
+            state: "absent"
+            zone: "{{ item.1.set.zone }}"
+            record: "{{ item.1.set.record }}"
+            type: "{{ item.1.set.type }}"
+            ttl: "{{ item.1.set.ttl }}"
+            value: ["{{ item.1.set.value }}"]
+            private_zone: "{{cluster_vars.route53_private_zone | default(true)}}"
+          with_nested:
+            - "{{ hosts_to_clean }}"
+            - "{{ r__route53_cname.results }}"
+          when: (item.1.set.value is defined)  and  ((item.0.name | regex_replace('-(?!.*-).*')) == (item.1.set.record | regex_replace('^(.*?)\\..*$', '\\1')))  and  (item.0.name == item.1.set.value | regex_replace('^(.*?)\\..*$', '\\1'))
+      when: cluster_vars.dns_server == "route53" and (use_new_route53 is not defined or use_new_route53|bool == false)
+
+
+    - name: "clean/dns/route53 | Delete DNS entries (new mechanism: route53_zone/ route53_info/ route53)"
       block:
         - name: clean/dns/route53 | Get Zone
           route53_zone:
@@ -110,7 +179,9 @@
             - "{{ hosts_to_clean }}"
             - "{{ records_to_clean }}"
           when: ((item.0.name | regex_replace('-(?!.*-).*')) == (item.1.Name | regex_replace('^(.*?)\\..*$', '\\1')))  and  ((item.0.name == item.1.ResourceRecords[0].Value | regex_replace('^(.*?)\\..*$', '\\1')))
-      when: cluster_vars.dns_server == "route53"
+      when: cluster_vars.dns_server == "route53" and (use_new_route53 is defined and use_new_route53|bool)
+
+
 
     - name: clean/dns/clouddns | Delete DNS entries
       block:
@@ -122,8 +193,6 @@
             service_account_file: "{{gcp_credentials_file}}"
           register: r__gcp_dns_managed_zone_info
 
-#        - debug: msg={{r__gcp_dns_managed_zone_info}}
-
         - name: clean/dns/clouddns | Get all non-peered DNS records for managed zones that match cluster_vars.dns_nameserver_zone
           gcp_dns_resource_record_set_info:
             auth_kind: serviceaccount
@@ -135,12 +204,8 @@
           register: r__gcp_dns_resource_record_set_info
           with_items: "{{r__gcp_dns_managed_zone_info.resources | json_query(\"[?dnsName==`\" + cluster_vars.dns_nameserver_zone + \"` && !(peeringConfig)]\") }}"
 
-#        - debug: msg={{r__gcp_dns_resource_record_set_info}}
-
         - name: clean/dns/clouddns | Delete A and CNAME records
           block:
-#            - debug: msg={{gcp_dns_resource_record_set_info__and__gcp_dns_managed_zone_info}}
-
             - name: clean/dns/clouddns | Delete A records
               gcp_dns_resource_record_set:
                 auth_kind: serviceaccount