Add Azure. Add 'free' ESXi (standalone host only, not vCentre)

EXAMPLE/Pipfile

@@ -16,6 +16,7 @@ google-auth = "*"
 google-api-python-client = "*"
 lxml = "*"
 apache-libcloud = "*"
+paramiko = "*"
 
 [dev-packages]
 

EXAMPLE/README.md

@@ -29,6 +29,16 @@ ansible-playbook cluster.yml -e buildenv=sandbox -e clusterid=testid -e cloud_ty
 ansible-playbook cluster.yml -e buildenv=sandbox -e clusterid=test_gcp_euw1 [email protected]
 ansible-playbook cluster.yml -e buildenv=sandbox -e clusterid=test_gcp_euw1 [email protected] --tags=clusterverse_clean -e clean=_all_
 ```
+### ESXi (free):
+```
+ansible-playbook cluster.yml -e buildenv=sandbox -e clusterid=testid -e cloud_type=esxifree -e region=homelab [email protected]
+ansible-playbook cluster.yml -e buildenv=sandbox -e clusterid=testid -e cloud_type=esxifree -e region=homelab [email protected] --tags=clusterverse_clean -e clean=_all_
+```
+### Azure:
+```
+ansible-playbook cluster.yml -e buildenv=sandbox -e clusterid=testid -e cloud_type=azure -e region=westeurope [email protected]
+ansible-playbook cluster.yml -e buildenv=sandbox -e clusterid=testid -e cloud_type=azure -e region=westeurope [email protected] --tags=clusterverse_clean -e clean=_all_
+```
 
 ### Mandatory command-line variables:
 + `-e buildenv=<sandbox>` - The environment (dev, stage, etc), which must be an attribute of `cluster_vars` (i.e. `{{cluster_vars[build_env]}}`)
@@ -70,6 +80,10 @@ ansible-playbook redeploy.yml -e buildenv=sandbox -e clusterid=test_aws_euw1 --v
 ansible-playbook redeploy.yml -e buildenv=sandbox -e clusterid=test -e cloud_type=gcp -e region=europe-west1 [email protected] -e canary=none
 ansible-playbook redeploy.yml -e buildenv=sandbox -e clusterid=test_aws_euw1 [email protected] -e canary=none
 ```
+### Azure:
+```
+ansible-playbook redeploy.yml -e buildenv=sandbox -e clusterid=test -e cloud_type=azure -e region=westeurope [email protected] -e canary=none
+```
 
 ### Mandatory command-line variables:
 + `-e buildenv=<sandbox>` - The environment (dev, stage, etc), which must be an attribute of `cluster_vars` defined in `group_vars/<clusterid>/cluster_vars.yml`

EXAMPLE/cluster_defs/azure/cluster_vars__cloud.yml

@@ -0,0 +1,20 @@
+---
+
+redeploy_schemes_supported: ['_scheme_addallnew_rmdisk_rollback', '_scheme_addnewvm_rmdisk_rollback', '_scheme_rmvm_rmdisk_only']     # TODO: support _scheme_rmvm_keepdisk_rollback
+
+cluster_vars:
+  dns_cloud_internal_domain: "ACCOUNTNAME_CHANGEME.onmicrosoft.com"        # The cloud-internal zone as defined by the cloud provider (e.g. GCP, AWS)
+  dns_server: ""                    # Specify DNS server. nsupdate, route53 or clouddns.  If empty string is specified, no DNS will be added.
+  assign_public_ip: "yes"
+  inventory_ip: "public"            # 'public' or 'private', (private in case we're operating in a private LAN).  If public, 'assign_public_ip' must be 'yes'
+  user_data: |-
+    #cloud-config
+    system_info:
+      default_user:
+        name: ansible
+  rules:
+    - name: "SSHExternal"
+      priority: "100"
+      protocol: "Tcp"
+      destination_port_range: ["22"]
+      source_address_prefix: "{{_ssh_whitelist}}"

EXAMPLE/cluster_defs/azure/testid/cluster_vars__clusterid.yml

@@ -0,0 +1,26 @@
+---
+
+prometheus_node_exporter_install: false
+filebeat_install: false
+metricbeat_install: false
+
+beats_config:
+  filebeat:
+#    output_logstash_hosts: ["localhost:5044"]        # The destination hosts for filebeat-gathered logs
+#    extra_logs_paths:                                # The array is optional, if you need to add more paths or files to scrape for logs
+#      - /var/log/myapp/*.log
+  metricbeat:
+#    output_logstash_hosts: ["localhost:5044"]        # The destination hosts for metricbeat-gathered metrics
+#    diskio:                                          # Diskio retrieves metrics for all disks partitions by default. When diskio.include_devices is defined, only look for defined partitions
+#      include_devices: ["sda", "sdb", "nvme0n1", "nvme1n1", "nvme2n1"]
+
+
+cluster_vars:
+  dns_nameserver_zone: &dns_nameserver_zone ""                   # The zone that dns_server will operate on.  gcloud dns needs a trailing '.'.  Leave blank if no external DNS (use IPs only)
+  dns_user_domain: "{%- if _dns_nameserver_zone -%}{{cloud_type}}-{{region}}.{{app_class}}.{{buildenv}}.{{_dns_nameserver_zone}}{%- endif -%}"         # A user-defined _domain_ part of the FDQN, (if more prefixes are required before the dns_nameserver_zone)
+  instance_profile_name: ""
+  custom_tagslabels:
+    inv_resident_id: "myresident"
+    inv_proposition_id: "myproposition"
+    inv_cost_centre: "0000000000"
+_dns_nameserver_zone: *dns_nameserver_zone

EXAMPLE/cluster_defs/azure/testid/westeurope/cluster_vars__region.yml

@@ -0,0 +1,9 @@
+---
+
+_ubuntu2004image: { "publisher": "canonical", "offer": "0001-com-ubuntu-server-focal", "sku": "20_04-lts-gen2", "version": "latest" }
+_ubuntu1804image: { "publisher": "canonical", "offer": "UbuntuServer", "sku": "18_04-lts-gen2", "version": "latest" }
+_centos7image:    { "publisher": "OpenLogic", "offer": "CentOS", "sku": "7_9-gen2", "version": "latest" }
+
+cluster_vars:
+  image: "{{_ubuntu2004image}}"
+

EXAMPLE/cluster_defs/azure/testid/westeurope/sandbox/cluster_vars__buildenv.yml

@@ -0,0 +1,69 @@
+---
+
+cluster_vars:
+  sandbox:
+    azure_subscription_id: !vault |
+      $ANSIBLE_VAULT;1.2;AES256;sandbox
+      7669080460651349243347331538721104778691266429457726036813912140404310
+    azure_client_id: !vault |
+      $ANSIBLE_VAULT;1.2;AES256;sandbox
+      7669080460651349243347331538721104778691266429457726036813912140404310
+    azure_secret: !vault |
+      $ANSIBLE_VAULT;1.2;AES256;sandbox
+      7669080460651349243347331538721104778691266429457726036813912140404310
+    azure_tenant: !vault |
+      $ANSIBLE_VAULT;1.2;AES256;sandbox
+      7669080460651349243347331538721104778691266429457726036813912140404310
+    ssh_connection_cfg:
+      host: &host_ssh_connection_cfg
+        ansible_user: "ansible"
+        ansible_ssh_private_key_file: !vault |
+          $ANSIBLE_VAULT;1.2;AES256;sandbox
+          7669080460651349243347331538721104778691266429457726036813912140404310
+      bastion:
+        ssh_args: '-o ProxyCommand="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ./id_rsa_bastion -W %h:%p -q [email protected]"'
+        ssh_priv_key: !vault |
+          $ANSIBLE_VAULT;1.2;AES256;sandbox
+          7669080460651349243347331538721104778691266429457726036813912140404310
+    azure_resource_group: "compute"
+    vnet_name: "{{buildenv}}"
+    vpc_subnet_name_prefix: "{{buildenv}}-test-{{region}}"
+#    nsupdate_cfg: {server: "", key_name: "", key_secret: ""}      # If you're using bind9 (or other nsupdate-compatible 'dns_server')
+
+    hosttype_vars:
+      sys:
+        auto_volumes: [ ]
+        flavor: Standard_B1ls
+        version: "{{sys_version | default('')}}"
+        vms_by_az: { 1: 1, 2: 0, 3: 0 }
+
+#      sysdisks2:
+#        auto_volumes:
+#          - { device_name: "0", disk_size_gb: 1, storage_account_type: "StandardSSD_LRS", mountpoint: "/media/mysvc0", fstype: "ext4", caching: "ReadOnly", perms: { owner: "root", group: "root", mode: "775" } }
+#          - { device_name: "1", disk_size_gb: 1, storage_account_type: "StandardSSD_LRS", mountpoint: "/media/mysvc1", fstype: "ext4", caching: "ReadOnly" }
+#        flavor: Standard_B1ls
+#        os_disk_size_gb: "35"              # This is optional, and if set, MUST be bigger than the original image size (e.g. 30GB for Ubuntu2004)
+#        version: "{{sysdisks_version | default('')}}"
+#        vms_by_az: { 1: 1, 2: 0, 3: 0 }
+
+#      sysdisks2lvm:
+#        auto_volumes:
+#          - { device_name: "0", disk_size_gb: 1, storage_account_type: "StandardSSD_LRS", mountpoint: "/media/mysvc0", fstype: "ext4", caching: "ReadOnly" }
+#          - { device_name: "1", disk_size_gb: 1, storage_account_type: "StandardSSD_LRS", mountpoint: "/media/mysvc0", fstype: "ext4", caching: "ReadOnly" }
+#        lvmparams: {vg_name: vg0, lv_name: lv0, lv_size: +100%FREE}
+#        flavor: Standard_B1ls
+#        os_disk_size_gb: "35"              # This is optional, and if set, MUST be bigger than the original image size (e.g. 30GB for Ubuntu2004)
+#        version: "{{sysdisks_version | default('')}}"
+#        vms_by_az: { 1: 1, 2: 0, 3: 0 }
+
+#      sysdisks4:
+#        auto_volumes:
+#          - { device_name: "3", disk_size_gb: 1, storage_account_type: "StandardSSD_LRS", mountpoint: "/media/mysvc3", fstype: "ext4", caching: "ReadOnly" }
+#          - { device_name: "1", disk_size_gb: 1, storage_account_type: "StandardSSD_LRS", mountpoint: "/media/mysvc1", fstype: "ext4", caching: "ReadOnly" }
+#          - { device_name: "0", disk_size_gb: 1, storage_account_type: "StandardSSD_LRS", mountpoint: "/media/mysvc0", fstype: "ext4", caching: "ReadOnly" }
+#          - { device_name: "2", disk_size_gb: 1, storage_account_type: "StandardSSD_LRS", mountpoint: "/media/mysvc2", fstype: "ext4", caching: "ReadOnly" }
+#        flavor: Standard_B2s    # B1ls only supports 2 disks (B2s supports 4)
+#        version: "{{sysdisks_version | default('')}}"
+#        vms_by_az: { 1: 1, 2: 1, 3: 0 }
+
+_host_ssh_connection_cfg: { <<: *host_ssh_connection_cfg }

EXAMPLE/cluster_defs/esxifree/cluster_vars__cloud.yml

@@ -0,0 +1,9 @@
+---
+
+_scheme_rmvm_keepdisk_rollback__copy_or_move: "move"
+
+cluster_vars:
+  dns_cloud_internal_domain: ""        # The cloud-internal zone as defined by the cloud provider (e.g. GCP, AWS)
+  dns_server: ""                       # Specify DNS server. nsupdate, route53 or clouddns.  If empty string is specified, no DNS will be added.
+  inventory_ip: "private"              # 'public' or 'private', (private in case we're operating in a private LAN).  If public, 'assign_public_ip' must be 'yes'
+  hardware_version: "19"

EXAMPLE/cluster_defs/esxifree/testid/cluster_vars__clusterid.yml

@@ -0,0 +1,25 @@
+---
+
+prometheus_node_exporter_install: false
+filebeat_install: false
+metricbeat_install: false
+
+beats_config:
+  filebeat:
+#    output_logstash_hosts: ["localhost:5044"]        # The destination hosts for filebeat-gathered logs
+#    extra_logs_paths:                                # The array is optional, if you need to add more paths or files to scrape for logs
+#      - /var/log/myapp/*.log
+  metricbeat:
+#    output_logstash_hosts: ["localhost:5044"]        # The destination hosts for metricbeat-gathered metrics
+#    diskio:                                          # Diskio retrieves metrics for all disks partitions by default. When diskio.include_devices is defined, only look for defined partitions
+#      include_devices: ["sda", "sdb", "nvme0n1", "nvme1n1", "nvme2n1"]
+
+
+cluster_vars:
+  dns_nameserver_zone: &dns_nameserver_zone ""        # The zone that dns_server will operate on.  gcloud dns needs a trailing '.'.  Leave blank if no external DNS (use IPs only)
+  dns_user_domain: "{%- if _dns_nameserver_zone -%}{{cloud_type}}-{{region}}.{{app_class}}.{{buildenv}}.{{_dns_nameserver_zone}}{%- endif -%}"         # A user-defined _domain_ part of the FDQN, (if more prefixes are required before the dns_nameserver_zone)
+  custom_tagslabels:
+    inv_resident_id: "myresident"
+    inv_proposition_id: "myproposition"
+    inv_cost_centre: "0000000000"
+_dns_nameserver_zone: *dns_nameserver_zone

EXAMPLE/cluster_defs/esxifree/testid/homelab/cluster_vars__region.yml

@@ -0,0 +1,13 @@
+---
+
+_ubuntu2004image: "gold-ubuntu2004l-20210415101808"
+_centos7image: "gold-ubuntu2004l-20210415101808"
+
+cluster_vars:
+  image: "{{_ubuntu2004image}}"
+  esxi_ip: "192.168.1.3"
+  username: "svc"
+  password: !vault |
+    $ANSIBLE_VAULT;1.1;AES256
+    7669080460651349243347331538721104778691266429457726036813912140404310
+  datastore: "4tb-evo860-ssd"

EXAMPLE/cluster_defs/esxifree/testid/homelab/sandbox/cluster_vars__buildenv.yml

@@ -0,0 +1,37 @@
+---
+
+cluster_vars:
+  sandbox:
+    ssh_connection_cfg:
+      host: &host_ssh_connection_cfg
+        ansible_user: "ansible"
+        ansible_ssh_private_key_file: !vault |
+          $ANSIBLE_VAULT;1.2;AES256;sandbox
+          7669080460651349243347331538721104778691266429457726036813912140404310
+#      bastion:
+#        ssh_args: '-o ProxyCommand="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ./id_rsa_bastion -W %h:%p -q [email protected]"'
+#        ssh_priv_key: !vault |
+#          $ANSIBLE_VAULT;1.2;AES256;sandbox
+#          7669080460651349243347331538721104778691266429457726036813912140404310
+    networks:
+      - networkName: "VM Network"
+        virtualDev: vmxnet3
+        cloudinit_netplan: { ethernets: { eth0: { dhcp4: true } } }
+#    nsupdate_cfg: {server: "", key_name: "", key_secret: ""}      # If you're using bind9 (or other nsupdate-compatible 'dns_server')
+
+    hosttype_vars:
+      sys:
+        auto_volumes: [ ]
+        flavor: { num_cpus: "2", memory_mb: "2048" }
+        version: "{{sys_version | default('')}}"
+        vms_by_az: { a: 1, b: 1, c: 0 }
+
+      sysdisks2:
+        auto_volumes:
+          - { mountpoint: "/media/mysvc1", volume_size: 1, provisioning_type: "thin", fstype: "ext4" }
+          - { mountpoint: "/media/mysvc2", volume_size: 1, provisioning_type: "thin", fstype: "ext4" }
+        flavor: { num_cpus: "2", memory_mb: "2048" }
+        version: "{{sys_version | default('')}}"
+        vms_by_az: { a: 1, b: 1, c: 0 }
+
+_host_ssh_connection_cfg: { <<: *host_ssh_connection_cfg }

EXAMPLE/jenkinsfiles/Jenkinsfile_ops

@@ -1,10 +1,10 @@
 #!groovy
 
 //These will not be needed if we're running this as a pipeline SCM job, as these are automatically added to the 'scm' variable, but if we instead just cut & paste this file into a pipeline job, they will be used as fallback
-def DEFAULT_CLUSTERVERSE_URL = "https://github.com/sky-uk/clusterverse"
-def DEFAULT_CLUSTERVERSE_BRANCH = "master"
+def DEFAULT_CLUSTERVERSE_URL = "https://github.com/dseeley/clusterverse"
+def DEFAULT_CLUSTERVERSE_BRANCH = "dps_esxi"
 
-def DEFAULT_CLUSTERVERSE_TESTSUITE_URL = "https://github.com/sky-uk/clusterverse-test"
+def DEFAULT_CLUSTERVERSE_TESTSUITE_URL = "https://github.com/dseeley/clusterverse_test"
 def DEFAULT_CLUSTERVERSE_TESTSUITE_BRANCH = "master"
 
 //This allows us to create our own Docker image for this specific use-case.  Once it is built, it will not be rebuilt, so only adds delay the first time we use it.
@@ -47,8 +47,8 @@ properties([
     parameters([
         string(name: 'APP_NAME', description: "An optional custom app_name to override the default in the playbook"),
         booleanParam(name: 'APPEND_BUILD_NUMBER', defaultValue: false, description: 'Tick the box to append the Jenkins BUILD_NUMBER to APP_NAME'),
-        choice(name: 'CLOUD_REGION', choices: ['aws/us-west-2', 'aws/eu-central-1', 'aws/eu-west-1', 'gcp/us-west2', 'gcp/europe-west1'], description: "Choose a cloud/region"),
-        choice(name: 'BUILDENV', choices: ['sandbox', 'dev', 'mgmt', 'tools', 'stage', 'prod'], description: "Choose an environment to deploy"),
+        choice(name: 'CLOUD_REGION', choices: ['esxifree/dougalab', 'aws/eu-west-1', 'gcp/europe-west1'], description: "Choose a cloud/region"),
+        choice(name: 'BUILDENV', choices: ['sandbox', 'dev', 'stage', 'prod'], description: "Choose an environment to deploy"),
         string(name: 'CLUSTER_ID', defaultValue: '', description: "Select a cluster_id to deploy", trim: true),
         booleanParam(name: 'DNS_FORCE_DISABLE', defaultValue: false, description: 'Tick the box to force disable the DNS as defined in playbook'),
         choice(name: 'DEPLOY_TYPE', choices: ['deploy', 'redeploy', 'clean'], description: "Choose the deploy type"),

README.md

@@ -1,6 +1,6 @@
 # clusterverse    [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) ![PRs Welcome](https://img.shields.io/badge/PRs-Welcome-brightgreen.svg)
 A full-lifecycle, immutable cloud infrastructure cluster management **role**, using Ansible.
-+ **Multi-cloud:** clusterverse can manage cluster lifecycle in AWS and GCP
++ **Multi-cloud:** clusterverse can manage cluster lifecycle in AWS, GCP, Free ESXi (standalone host only, not vCentre) and Azure
 + **Deploy:**  You define your infrastructure as code (in Ansible yaml), and clusterverse will deploy it 
 + **Scale-up:**  If you change the cluster definitions and rerun the deploy, new nodes will be added.
 + **Redeploy (e.g. up-version):** If you need to up-version, or replace the underlying OS, (i.e. to achieve fully immutable, zero-patching redeploys), the `redeploy.yml` playbook will replace each node in the cluster (via various redeploy schemes), and rollback if any failures occur. 
@@ -38,6 +38,22 @@ To active the pipenv:
   + During execution, the json file will be copied locally because the Ansible GCP modules often require the file as input. 
 + Google Cloud SDK needs to be installed to run gcloud command-line (e.g. to disable delete protection) - this is handled by `pipenv install`
 
+### ESXi (free)
++ Username & password for a privileged user on an ESXi host
++ SSH must be enabled on the host
++ Set the `Config.HostAgent.vmacore.soap.maxSessionCount` variable to 0 to allow many concurrent tests to run.   
+
+### Azure
++ Create an Azure account.
++ Create a Tenant and a Subscription
++ Create a Resource group and networks/subnetworks within that.
++ Create a service principal - add the credentials to:
+  + `cluster_vars[buildenv].azure_subscription_id`
+  + `cluster_vars[buildenv].azure_client_id`
+  + `cluster_vars[buildenv].azure_secret`
+  + `cluster_vars[buildenv].azure_tenant`
+
+
 ### DNS
 DNS is optional.  If unset, no DNS names will be created.  If DNS is required, you will need a DNS zone delegated to one of the following:
 + nsupdate (e.g. bind9)
@@ -227,3 +243,4 @@ The role is designed to run in two modes:
       + If `canary=start`, only the first node is redeployed.  If `canary=finish`, only the remaining (non-first), nodes are replaced.  If `canary=none`, all nodes are redeployed.
       + If the process fails for any reason, the old VMs are reinstated (and the disks reattached to the old nodes), and the new VMs are stopped (rollback)
       + To delete the old VMs, either set '-e canary_tidy_on_success=true', or call redeploy.yml with '-e canary=tidy'
+      + (Azure functionality coming soon)