[FEATURE] Allow clusterverse to be dynamically acquired via ansible-g…

…alaxy as part of the playbook. (#70)

EXAMPLE/README.md

@@ -67,11 +67,13 @@ ansible-playbook -u <username> --private-key=/home/<user>/.ssh/<rsa key> cluster
 + `-e app_class=<proxy>` - Normally defined in `group_vars/<clusterid>/cluster_vars.yml`.  The class of application (e.g. 'database', 'webserver'); becomes part of the fqdn
 + `-e release_version=<v1.0.1>` - Identifies the application version that is being deployed.
 + `-e clean=[current|retiring|redeployfail|_all_]` - Deletes VMs in `lifecycle_state`, or `_all_`, as well as networking and security groups
-+ `-e do_package_upgrade=true` - Upgrade the OS packages (not good for determinism)
++ `-e pkgupdate=[always|onCreate]` - Upgrade the OS packages (not good for determinism).  `onCreate` only upgrades when creating the VM for the first time.
 + `-e reboot_on_package_upgrade=true` - After updating packages, performs a reboot on all nodes.
 + `-e prometheus_node_exporter_install=false` - Does not install the prometheus node_exporter
 + `-e static_journal=true` - Creates /var/log/journal directory, which will keep a permanent record of journald logs in systemd machines (normally ephemeral)
 + `-e filebeat_install=false` - Does not install filebeat
++ `-e metricbeat_install=false` - Does not install metricbeat
++ `-e wait_for_dns=false` - Does not wait for DNS resolution
 + `-e create_gcp_network=true` - Create GCP network and subnetwork (probably needed if creating from scratch and using public network)
 
 ### Tags
@@ -83,7 +85,7 @@ ansible-playbook -u <username> --private-key=/home/<user>/.ssh/<rsa key> cluster
 ---
 
 ## Invocation examples: _redeploy_
-The `redeploy.yml` sub-role will completely redeploy the cluster; this is useful for example to upgrade the underlying operating system version.
+The `redeploy.yml` sub-role will completely redeploy the cluster; this is useful for example to upgrade the underlying operating system version, or changing the disk sizes.
 
 ### AWS:
 ```

EXAMPLE/cluster.yml

@@ -3,22 +3,28 @@
 - name: Deploy the cluster
   hosts: localhost
   connection: local
-  roles:
-    - { role: clusterverse/clean, tags: [clusterverse_clean], when: clean is defined }
-    - { role: clusterverse/create, tags: [clusterverse_create] }
-    - { role: clusterverse/dynamic_inventory, tags: [clusterverse_dynamic_inventory] }
+  gather_facts: no
+  tasks:
+    - { name: "Get dependent roles via ansible-galaxy", local_action: "command ansible-galaxy install -fr requirements.yml", tags: ["always"] }
+
+    - { include_role: { name: "clusterverse/clean", apply: {tags: &roletag_clean  ["clusterverse_clean"]} }, tags:  *roletag_clean, when: "clean is defined" }
+    - { include_role: { name: "clusterverse/create", apply: {tags: &roletag_create  ["clusterverse_create"]} }, tags:  *roletag_create }
+    - { include_role: { name: "clusterverse/dynamic_inventory", apply: {tags: &roletag_dynamic_inventory  ["clusterverse_dynamic_inventory"]} }, tags:  *roletag_dynamic_inventory }
 
 - name: Configure the cluster
   hosts: all
-  roles: [ { role: clusterverse/config, tags: [clusterverse_config] } ]
+  tasks:
+    - { include_role: { name: "clusterverse/config", apply: {tags: &roletag_config  ["clusterverse_config"]} }, tags:  *roletag_config }
 
 ## Application roles
-- name: Test application role
+- name: Application roles
   hosts: all
-  roles: [ { role: testrole, tags: [testrole] } ]
+  tasks:
+    - { include_role: { name: "testrole", apply: {tags: &roletag_testrole  ["testrole"]} }, tags:  *roletag_testrole }
 ##
 
 - name: Perform cluster readiness operations
   hosts: localhost
   connection: local
-  roles: [ { role: clusterverse/readiness, tags: [clusterverse_readiness] } ]
+  tasks:
+    - { include_role: { name: "clusterverse/readiness", apply: {tags: &roletag_readiness  ["clusterverse_readiness"]} }, tags:  *roletag_readiness }

EXAMPLE/group_vars/_skel/cluster_vars.yml

@@ -35,10 +35,7 @@ cloud_agent:
 
 ## Bind configuration and credentials, per environment
 bind9:
-  sandbox:
-    server:
-    key_name:
-    key_secret:
+  sandbox: {server: "", key_name: "", key_secret: ""}
 
 cluster_name: "{{app_name}}-{{buildenv}}"       # Identifies the cluster within the cloud environment
 

EXAMPLE/group_vars/test_aws_euw1/cluster_vars.yml

@@ -1,8 +1,8 @@
 ---
 
-redeploy_scheme: _scheme_addallnew_rmdisk_rollback
+#redeploy_scheme: _scheme_addallnew_rmdisk_rollback
 #redeploy_scheme: _scheme_addnewvm_rmdisk_rollback
-#redeploy_scheme: _scheme_rmvm_rmdisks_only
+#redeploy_scheme: _scheme_rmvm_rmdisk_only
 
 app_name: "test"                  # The name of the application cluster (e.g. 'couchbase', 'nginx'); becomes part of cluster_name.
 app_class: "test"                 # The class of application (e.g. 'database', 'webserver'); becomes part of the fqdn
@@ -35,10 +35,7 @@ cloud_agent:
 
 ## Bind configuration and credentials, per environment
 bind9:
-  sandbox:
-    server:
-    key_name:
-    key_secret:
+  sandbox: {server: "", key_name: "", key_secret: ""}
 
 cluster_name: "{{app_name}}-{{buildenv}}"       # Identifies the cluster within the cloud environment
 

EXAMPLE/group_vars/test_gcp_euw1/cluster_vars.yml

@@ -4,9 +4,9 @@
 gcp_credentials_file: "{{ lookup('env','GCP_CREDENTIALS') | default('/dev/null', true) }}"
 gcp_credentials_json: "{{ lookup('file', gcp_credentials_file) | default({'project_id': 'GCP_CREDENTIALS__NOT_SET','client_email': 'GCP_CREDENTIALS__NOT_SET'}, true) }}"
 
-redeploy_scheme: _scheme_addallnew_rmdisk_rollback
+#redeploy_scheme: _scheme_addallnew_rmdisk_rollback
 #redeploy_scheme: _scheme_addnewvm_rmdisk_rollback
-#redeploy_scheme: _scheme_rmvm_rmdisks_only
+#redeploy_scheme: _scheme_rmvm_rmdisk_only
 
 app_name: "test"                  # The name of the application cluster (e.g. 'couchbase', 'nginx'); becomes part of cluster_name.
 app_class: "test"                 # The class of application (e.g. 'database', 'webserver'); becomes part of the fqdn
@@ -39,10 +39,7 @@ cloud_agent:
 
 ## Bind configuration and credentials, per environment
 bind9:
-  sandbox:
-    server:
-    key_name:
-    key_secret:
+  sandbox: {server: "", key_name: "", key_secret: ""}
 
 cluster_name: "{{app_name}}-{{buildenv}}"       # Identifies the cluster within the cloud environment
 

EXAMPLE/redeploy.yml

@@ -4,8 +4,12 @@
   hosts: localhost
   connection: local
   tasks:
+    - name: "Get dependent roles via ansible-galaxy"
+      local_action: "command ansible-galaxy install -fr requirements.yml"
+      tags: ["always"]
+
     - name: Run redeploy
-      import_role:
+      include_role:
         name: clusterverse/redeploy
       vars:
         mainclusteryml: "cluster.yml"

_dependencies/filter_plugins/custom.py

@@ -41,14 +41,18 @@ def iplookup(fqdn):
 def json_loads_loose(inStr):
     import re, json
 
-    display.vv(u"json_loads_loose - input type: %s" % type(inStr))
+    display.vvv(u"json_loads_loose - input type: %s" % type(inStr))
     if type(inStr) is dict or type(inStr) is list:
         json_object = json.loads((str(json.dumps(inStr))).encode('utf-8'))
     else:
         try:
             json_object = json.loads(inStr)
         except (ValueError, AttributeError) as e:
-            return json.loads(str(re.sub(r'\'(.*?)\'([,:}])', r'"\1"\2', inStr).replace(': True', ': "True"').replace(': False', ': "False"')).encode('utf-8'))
+            try:
+                json_object = json.loads(str(re.sub(r'\'(.*?)\'([,:}])', r'"\1"\2', inStr).replace(': True', ': "True"').replace(': False', ': "False"')).encode('utf-8'))
+            except (ValueError, AttributeError) as e:
+                display.v(u"json_loads_loose - WARNING: could not parse attribute string as json: %s" % inStr)
+                return inStr
     return json_object
 
 
@@ -57,6 +61,5 @@ def filters(self):
         return {
             'dict_agg': dict_agg,
             'iplookup': iplookup,
-
             'json_loads_loose': json_loads_loose
         }

_dependencies/vars_plugins/cli_facts.py

@@ -0,0 +1,30 @@
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+ANSIBLE_METADATA = {'metadata_version': '1.2',
+                    'status': ['preview'],
+                    'supported_by': 'community'}
+
+DOCUMENTATION = '''
+---
+cars: cli_facts
+short_description: Expose the system ARGV and CLI arguments as facts in plays.
+version_added: "2.8"
+author: "Dougal Seeley"
+description:
+    - Expose the system ARGV and CLI arguments as facts in plays.  Two new facts are added: argv and cliargs.
+options:
+requirements:
+'''
+
+from ansible.plugins.vars import BaseVarsPlugin
+from ansible.context import CLIARGS
+import sys
+
+
+class VarsModule(BaseVarsPlugin):
+    REQUIRES_WHITELIST = False
+
+    def get_vars(self, loader, path, entities, cache=True):
+        super(VarsModule, self).get_vars(loader, path, entities)
+        return {"cliargs": dict(CLIARGS), "argv": sys.argv}

cluster_hosts/tasks/get_cluster_hosts_target.yml

@@ -101,7 +101,7 @@
   when: cluster_vars.type == "aws"
 
 
-- name: get_cluster_hosts_target/gcp | GCP-specific modifications to cluster_hosts_target - add rootvol size
+- name: get_cluster_hosts_target/gcp | GCP-specific modifications to cluster_hosts_target
   block:
     - name: get_cluster_hosts_target/gcp | Update cluster_hosts_target with rootvol_size
       set_fact:

cluster_hosts/tasks/main.yml

@@ -20,9 +20,9 @@
         cluster_suffix: "{{cluster_suffixes_current[0]}}"
       when: cluster_suffixes_current | unique | length==1
 
-    - name: Create new cluster_suffix
+    - name: Create new cluster_suffix (epoch time)
       set_fact:
-        cluster_suffix: "{{ansible_date_time.epoch}}"
+        cluster_suffix: "{{ lookup('pipe', 'date +%s') }}"
       when: cluster_suffixes_current | unique | length==0
 
     - debug: msg="cluster_suffix = {{cluster_suffix}}"

config/tasks/create_dns_a.yml

@@ -1,52 +1,57 @@
 ---
 
-- name: config/dns/a/nsupdate | create/update A records in bind (nsupdate)
-  nsupdate:
-    key_name: "{{bind9[buildenv].key_name}}"
-    key_secret: "{{bind9[buildenv].key_secret}}"
-    server: "{{bind9[buildenv].server}}"
-    ttl: 60
-    zone: "{{cluster_vars.dns_nameserver_zone}}"
-    record: "{{item.hostname}}.{{cluster_vars.dns_user_domain | regex_replace('^(.*?)\\.' + cluster_vars.dns_nameserver_zone, '\\1')}}"
-    value: "{{ hostvars[item.hostname]['ansible_host'] }}"
-  become: false
-  delegate_to: localhost
-  run_once: true
-  with_items: "{{ cluster_hosts_target }}"
+- block:
+    - name: config/dns/a/nsupdate | create/update A records in bind (nsupdate)
+      nsupdate:
+        key_name: "{{bind9[buildenv].key_name}}"
+        key_secret: "{{bind9[buildenv].key_secret}}"
+        server: "{{bind9[buildenv].server}}"
+        ttl: 60
+        zone: "{{cluster_vars.dns_nameserver_zone}}"
+        record: "{{item.hostname}}.{{cluster_vars.dns_user_domain | regex_replace('^(.*?)\\.' + cluster_vars.dns_nameserver_zone, '\\1')}}"
+        value: "{{ hostvars[item.hostname]['ansible_host'] }}"
+      become: false
+      delegate_to: localhost
+      run_once: true
+      with_items: "{{ cluster_hosts_target }}"
+
+    - name: config/dns/a/nsupdate | Wait for a short delay to allow zone transfers to complete (help prevent negative cache)
+      pause:
+        seconds: 10
   when: cluster_vars.dns_server == "nsupdate"
 
-- name: config/dns/a/route53 | create/update A records in AWS (route53)
-  route53:
-    aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}"
-    aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}"
-    state: present
-    zone: "{{cluster_vars.dns_nameserver_zone}}"
-    record: "{{item.hostname}}.{{cluster_vars.dns_user_domain}}"
-    type: A
-    ttl: 60
-    value: "{{ hostvars[item.hostname]['ansible_host'] }}"
-    private_zone: "{{cluster_vars.route53_private_zone | default(true)}}"
-    overwrite: true
-    wait: yes
-  become: false
-  delegate_to: localhost
-  run_once: true
-  with_items: "{{ cluster_hosts_target }}"
-  when: cluster_vars.dns_server=="route53"
-  async: 7200
-  poll: 0
-  register: route53_records
+- block:
+    - name: config/dns/a/route53 | create/update A records in AWS (route53)
+      route53:
+        aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}"
+        aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}"
+        state: present
+        zone: "{{cluster_vars.dns_nameserver_zone}}"
+        record: "{{item.hostname}}.{{cluster_vars.dns_user_domain}}"
+        type: A
+        ttl: 60
+        value: "{{ hostvars[item.hostname]['ansible_host'] }}"
+        private_zone: "{{cluster_vars.route53_private_zone | default(true)}}"
+        overwrite: true
+        wait: yes
+      become: false
+      delegate_to: localhost
+      run_once: true
+      with_items: "{{ cluster_hosts_target }}"
+      async: 7200
+      poll: 0
+      register: r__route53
 
-- name: config/dns/a/route53 | Wait for records to be replicated to all Amazon Route 53 DNS servers
-  async_status:
-    jid: "{{ item.ansible_job_id }}"
-  register: route53_jobs
-  until: route53_jobs.finished
-  delay: 1
-  retries: 300
-  run_once: true
-  with_items: "{{route53_records.results}}"
-  delegate_to: localhost
+    - name: config/dns/a/route53 | Wait for records to be replicated to all Amazon Route 53 DNS servers
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: route53_jobs
+      until: route53_jobs.finished
+      delay: 1
+      retries: 300
+      run_once: true
+      with_items: "{{r__route53.results}}"
+      delegate_to: localhost
   when: cluster_vars.dns_server=="route53"
 
 - name: config/dns/a/clouddns | create/update A records in GCP (clouddns)

config/tasks/main.yml

@@ -80,9 +80,9 @@
   include_tasks: cloud_agents.yml
   when: (cloud_agent is defined and cloud_agent)
 
-- name: Update packages (when do_package_upgrade is defined)
+- name: Update packages (when pkgupdate is defined)
   include_tasks: pkgupdate.yml
-  when: do_package_upgrade is defined and do_package_upgrade|bool
+  when: pkgupdate is defined and (pkgupdate == 'always' or (pkgupdate == 'onCreate' and inventory_hostname in (hostvars['localhost'].cluster_hosts_created | json_query('[].hostname'))))
 
 - name: Set hostname (e.g. AWS doesn't set it automatically)
   become: true