diff --git a/EXAMPLE/cluster_defs/esxifree/testid/homelab/cluster_vars__region.yml b/EXAMPLE/cluster_defs/esxifree/testid/homelab/cluster_vars__region.yml index 6873eb31..3bc0fdd9 100644 --- a/EXAMPLE/cluster_defs/esxifree/testid/homelab/cluster_vars__region.yml +++ b/EXAMPLE/cluster_defs/esxifree/testid/homelab/cluster_vars__region.yml @@ -1,7 +1,7 @@ --- -_ubuntu2004image: "gold-ubuntu2004l-20210415101808" -_centos7image: "gold-ubuntu2004l-20210415101808" +_ubuntu2004image: "gold-ubuntu2004l-20210610082838" +_centos7image: "gold-ubuntu2004l-20210610082838" cluster_vars: image: "{{_ubuntu2004image}}" diff --git a/EXAMPLE/cluster_defs/esxifree/testid/homelab/sandbox/cluster_vars__buildenv.yml b/EXAMPLE/cluster_defs/esxifree/testid/homelab/sandbox/cluster_vars__buildenv.yml index 51a46157..3f394ce4 100644 --- a/EXAMPLE/cluster_defs/esxifree/testid/homelab/sandbox/cluster_vars__buildenv.yml +++ b/EXAMPLE/cluster_defs/esxifree/testid/homelab/sandbox/cluster_vars__buildenv.yml @@ -21,7 +21,7 @@ cluster_vars: flavor: { num_cpus: "2", memory_mb: "2048" } version: "{{sys_version | default('')}}" networks: [ &sys_NET1 { networkName: "VM Network", virtualDev: vmxnet3 } ] - vms_by_az: { a: 2, b: 1, c: 0 } + vms_by_az: { a: 1, b: 1, c: 0 } sysdisks2: auto_volumes: diff --git a/EXAMPLE/jenkinsfiles/Jenkinsfile_ops b/EXAMPLE/jenkinsfiles/Jenkinsfile_ops index acf88e65..9d3ff6e5 100644 --- a/EXAMPLE/jenkinsfiles/Jenkinsfile_ops +++ b/EXAMPLE/jenkinsfiles/Jenkinsfile_ops @@ -2,7 +2,7 @@ //These will not be needed if we're running this as a pipeline SCM job, as these are automatically added to the 'scm' variable, but if we instead just cut & paste this file into a pipeline job, they will be used as fallback def DEFAULT_CLUSTERVERSE_URL = "https://github.com/dseeley/clusterverse" -def DEFAULT_CLUSTERVERSE_BRANCH = "dps_esxi" +def DEFAULT_CLUSTERVERSE_BRANCH = "master" def DEFAULT_CLUSTERVERSE_TESTSUITE_URL = "https://github.com/dseeley/clusterverse_test" def DEFAULT_CLUSTERVERSE_TESTSUITE_BRANCH = "master" diff --git a/README.md b/README.md index 4e43090d..3d99353f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # clusterverse   [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) ![PRs Welcome](https://img.shields.io/badge/PRs-Welcome-brightgreen.svg) A full-lifecycle, immutable cloud infrastructure cluster management **role**, using Ansible. -+ **Multi-cloud:** clusterverse can manage cluster lifecycle in AWS, GCP, Free ESXi (standalone host only, not vCentre) and Azure ++ **Multi-cloud:** clusterverse can manage cluster lifecycle in AWS, GCP, Azure and Free ESXi (standalone host only, not vCentre). + **Deploy:** You define your infrastructure as code (in Ansible yaml), and clusterverse will deploy it + **Scale-up:** If you change the cluster definitions and rerun the deploy, new nodes will be added. + **Redeploy (e.g. up-version):** If you need to up-version, or replace the underlying OS, (i.e. to achieve fully immutable, zero-patching redeploys), the `redeploy.yml` playbook will replace each node in the cluster (via various redeploy schemes), and rollback if any failures occur. diff --git a/_dependencies/tasks/main.yml b/_dependencies/tasks/main.yml index 6c4b524e..3839e253 100644 --- a/_dependencies/tasks/main.yml +++ b/_dependencies/tasks/main.yml @@ -30,7 +30,14 @@ - name: Preflight check block: - - assert: { that: "ansible_version.full is version_compare('2.10', '>=')", fail_msg: "Ansible > 2.10 required for Azure support." } + - assert: { that: "ansible_version.full is version_compare('2.10', '>=')", fail_msg: "ansible-core > 2.10 required for Azure support." } + when: cluster_vars.type == "azure" + + - assert: + that: "(ansible_version.full is version_compare('2.9.6', '>=') and ansible_version.full is version_compare('2.10.6', '<=')) or galaxy_collections['community.aws'].version is version_compare('1.5.0', '>=')" + fail_msg: "If Ansible > 2.9.6 then community.aws > 1.5.0 is required for valid community.aws.route53 support (by default in Ansible v4)." + vars: { galaxy_collections: "{{ (lookup('pipe', 'ansible-galaxy collection list --format json')) | from_json | json_query(\"* | [0]\") }}" } + - assert: { that: "app_name is defined and app_name != ''", fail_msg: "Please define app_name" } - assert: { that: "app_class is defined and app_class != ''", fail_msg: "Please define app_class" } - assert: { that: "cluster_vars is defined", fail_msg: "Please define cluster_vars" } diff --git a/clean/tasks/dns.yml b/clean/tasks/dns.yml index 7b1b7669..5da2f36a 100644 --- a/clean/tasks/dns.yml +++ b/clean/tasks/dns.yml @@ -35,81 +35,71 @@ - name: clean/dns/route53 | Delete DNS entries block: - - name: clean/dns/route53 | Get Zone - route53_zone: - aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}" - aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}" - zone: "{{cluster_vars.dns_nameserver_zone}}" - vpc_id: "{{ vpc_id if cluster_vars.route53_private_zone|bool else omit }}" # If this is a private zone, we need to request the private (VPC/Region-specific) version of the Zone - vpc_region: "{{ cluster_vars.region if cluster_vars.route53_private_zone|bool else omit }}" # If this is a private zone, we need to request the private (VPC/Region-specific) version of the Zone - register: r__route53_zone - - # Note: route53_info currently does not honour the 'max_items' or 'type' fields, (and if 'start_record_name' is not found, it just returns all records), so we need to filter the responses to match 'hosts_to_clean' when doing the delete - # Note: cannot run route53_info asynchronously as it makes too many concurrent requests and blows the AWS Route53 API limit. - name: clean/dns/route53 | Get A records - route53_info: + route53: aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}" aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}" + state: "get" + zone: "{{cluster_vars.dns_nameserver_zone}}" + record: "{{item.name}}.{{cluster_vars.dns_user_domain}}" type: "A" - max_items: 1 - query: record_sets - hosted_zone_id: "{{ r__route53_zone.zone_id }}" - start_record_name: "{{item.name}}.{{cluster_vars.dns_user_domain}}" - register: r__route53_info - until: r__route53_info is success - retries: 10 + private_zone: "{{cluster_vars.route53_private_zone | default(true)}}" + register: r__route53_a with_items: "{{ hosts_to_clean }}" + ignore_errors: yes + - name: clean/dns/route53 | Remove failed DNS lookups from route53 state=get + set_fact: + r__route53_a: "{{r__route53_a.results | selectattr('failed', '==', false) | list}}" + when: r__route53_a is failed + + - debug: msg={{r__route53_a}} - name: clean/dns/route53 | Delete A records route53: aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}" aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}" state: "absent" - zone: "{{ cluster_vars.dns_nameserver_zone }}" - record: "{{ item.Name }}" - type: "{{ item.Type }}" - ttl: "{{ item.TTL }}" - value: "{{ item.ResourceRecords | json_query(\"[].Value\") }}" + zone: "{{ item.set.zone }}" + record: "{{ item.set.record }}" + type: "{{ item.set.type }}" + ttl: "{{ item.set.ttl }}" + value: ["{{ item.set.value }}"] private_zone: "{{cluster_vars.route53_private_zone | default(true)}}" - with_items: "{{ records_to_clean }}" - vars: - _hostnames_to_clean: "{{ hosts_to_clean | json_query(\"[].name\") | map('regex_replace', '^(.*)$', '\\1.' + cluster_vars.dns_user_domain + '.') | list }}" - records_to_clean: "{{ r__route53_info.results | json_query(\"[].ResourceRecordSets[?Type=='A' && contains(\"+ _hostnames_to_clean | string +\", Name)][]\") | unique }}" + with_items: "{{ r__route53_a.results }}" + when: item.set.value is defined - # Note: route53_info currently does not honour the 'max_items' or 'type' fields, (and if 'start_record_name' is not found, it just returns all records), so we need to filter the responses to match 'hosts_to_clean' when doing the delete - # Note: cannot run route53_info asynchronously as it makes too many concurrent requests and blows the AWS Route53 API limit. - name: clean/dns/route53 | Get CNAME records - route53_info: + route53: aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}" aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}" + state: "get" + zone: "{{cluster_vars.dns_nameserver_zone}}" + record: "{{item.name | regex_replace('-(?!.*-).*')}}.{{cluster_vars.dns_user_domain}}" type: "CNAME" - max_items: 1 - query: record_sets - hosted_zone_id: "{{ r__route53_zone.zone_id }}" - start_record_name: "{{item.name | regex_replace('-(?!.*-).*')}}.{{cluster_vars.dns_user_domain}}" - register: r__route53_info + private_zone: "{{cluster_vars.route53_private_zone | default(true)}}" + register: r__route53_cname with_items: "{{ hosts_to_clean }}" - until: r__route53_info is success - retries: 10 + ignore_errors: yes + - name: clean/dns/route53 | Remove failed DNS lookups from route53 state=get + set_fact: + r__route53_cname: "{{r__route53_cname.results | selectattr('failed', '==', false) | list}}" + when: r__route53_cname is failed - name: clean/dns/route53 | Delete CNAME records route53: aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}" aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}" state: "absent" - zone: "{{ cluster_vars.dns_nameserver_zone }}" - record: "{{ item.1.Name }}" - type: "{{ item.1.Type }}" - ttl: "{{ item.1.TTL }}" - value: "{{ item.1.ResourceRecords | json_query(\"[].Value\") }}" + zone: "{{ item.1.set.zone }}" + record: "{{ item.1.set.record }}" + type: "{{ item.1.set.type }}" + ttl: "{{ item.1.set.ttl }}" + value: ["{{ item.1.set.value }}"] private_zone: "{{cluster_vars.route53_private_zone | default(true)}}" - vars: - _cnames_to_clean: "{{ hosts_to_clean | json_query(\"[].name\") | map('regex_replace', '^(.*)-(?!.*-).*$', '\\1.' + cluster_vars.dns_user_domain + '.') | list }}" #Remove the last '-.*' (cluster_suffix) - records_to_clean: "{{ r__route53_info.results | json_query(\"[].ResourceRecordSets[?Type=='CNAME' && contains(\"+ _cnames_to_clean | string +\", Name)][]\") | unique }}" with_nested: - "{{ hosts_to_clean }}" - - "{{ records_to_clean }}" - when: ((item.0.name | regex_replace('-(?!.*-).*')) == (item.1.Name | regex_replace('^(.*?)\\..*$', '\\1'))) and ((item.0.name == item.1.ResourceRecords[0].Value | regex_replace('^(.*?)\\..*$', '\\1'))) + - "{{ r__route53_cname.results }}" + when: (item.1.set.value is defined) and ((item.0.name | regex_replace('-(?!.*-).*')) == (item.1.set.record | regex_replace('^(.*?)\\..*$', '\\1'))) and (item.0.name == item.1.set.value | regex_replace('^(.*?)\\..*$', '\\1')) when: cluster_vars.dns_server == "route53" - name: clean/dns/clouddns | Delete DNS entries