diff --git a/EXAMPLE/Pipfile b/EXAMPLE/Pipfile index 49dc09fd..e40fb9f6 100644 --- a/EXAMPLE/Pipfile +++ b/EXAMPLE/Pipfile @@ -16,6 +16,7 @@ google-auth = "*" google-api-python-client = "*" lxml = "*" apache-libcloud = "*" +paramiko = "*" [dev-packages] diff --git a/EXAMPLE/README.md b/EXAMPLE/README.md index 9e5a5527..b7fb7276 100644 --- a/EXAMPLE/README.md +++ b/EXAMPLE/README.md @@ -29,6 +29,16 @@ ansible-playbook cluster.yml -e buildenv=sandbox -e clusterid=testid -e cloud_ty ansible-playbook cluster.yml -e buildenv=sandbox -e clusterid=test_gcp_euw1 --vault-id=sandbox@.vaultpass-client.py ansible-playbook cluster.yml -e buildenv=sandbox -e clusterid=test_gcp_euw1 --vault-id=sandbox@.vaultpass-client.py --tags=clusterverse_clean -e clean=_all_ ``` +### ESXi (free): +``` +ansible-playbook cluster.yml -e buildenv=sandbox -e clusterid=testid -e cloud_type=esxifree -e region=homelab --vault-id=sandbox@.vaultpass-client.py +ansible-playbook cluster.yml -e buildenv=sandbox -e clusterid=testid -e cloud_type=esxifree -e region=homelab --vault-id=sandbox@.vaultpass-client.py --tags=clusterverse_clean -e clean=_all_ +``` +### Azure: +``` +ansible-playbook cluster.yml -e buildenv=sandbox -e clusterid=testid -e cloud_type=azure -e region=westeurope --vault-id=sandbox@.vaultpass-client.py +ansible-playbook cluster.yml -e buildenv=sandbox -e clusterid=testid -e cloud_type=azure -e region=westeurope --vault-id=sandbox@.vaultpass-client.py --tags=clusterverse_clean -e clean=_all_ +``` ### Mandatory command-line variables: + `-e buildenv=` - The environment (dev, stage, etc), which must be an attribute of `cluster_vars` (i.e. `{{cluster_vars[build_env]}}`) @@ -70,6 +80,10 @@ ansible-playbook redeploy.yml -e buildenv=sandbox -e clusterid=test_aws_euw1 --v ansible-playbook redeploy.yml -e buildenv=sandbox -e clusterid=test -e cloud_type=gcp -e region=europe-west1 --vault-id=sandbox@.vaultpass-client.py -e canary=none ansible-playbook redeploy.yml -e buildenv=sandbox -e clusterid=test_aws_euw1 --vault-id=sandbox@.vaultpass-client.py -e canary=none ``` +### Azure: +``` +ansible-playbook redeploy.yml -e buildenv=sandbox -e clusterid=test -e cloud_type=azure -e region=westeurope --vault-id=sandbox@.vaultpass-client.py -e canary=none +``` ### Mandatory command-line variables: + `-e buildenv=` - The environment (dev, stage, etc), which must be an attribute of `cluster_vars` defined in `group_vars//cluster_vars.yml` diff --git a/EXAMPLE/cluster_defs/azure/cluster_vars__cloud.yml b/EXAMPLE/cluster_defs/azure/cluster_vars__cloud.yml new file mode 100644 index 00000000..e8a6e713 --- /dev/null +++ b/EXAMPLE/cluster_defs/azure/cluster_vars__cloud.yml @@ -0,0 +1,20 @@ +--- + +redeploy_schemes_supported: ['_scheme_addallnew_rmdisk_rollback', '_scheme_addnewvm_rmdisk_rollback', '_scheme_rmvm_rmdisk_only'] # TODO: support _scheme_rmvm_keepdisk_rollback + +cluster_vars: + dns_cloud_internal_domain: "ACCOUNTNAME_CHANGEME.onmicrosoft.com" # The cloud-internal zone as defined by the cloud provider (e.g. GCP, AWS) + dns_server: "" # Specify DNS server. nsupdate, route53 or clouddns. If empty string is specified, no DNS will be added. + assign_public_ip: "yes" + inventory_ip: "public" # 'public' or 'private', (private in case we're operating in a private LAN). If public, 'assign_public_ip' must be 'yes' + user_data: |- + #cloud-config + system_info: + default_user: + name: ansible + rules: + - name: "SSHExternal" + priority: "100" + protocol: "Tcp" + destination_port_range: ["22"] + source_address_prefix: "{{_ssh_whitelist}}" diff --git a/EXAMPLE/cluster_defs/azure/testid/cluster_vars__clusterid.yml b/EXAMPLE/cluster_defs/azure/testid/cluster_vars__clusterid.yml new file mode 100644 index 00000000..4156457b --- /dev/null +++ b/EXAMPLE/cluster_defs/azure/testid/cluster_vars__clusterid.yml @@ -0,0 +1,26 @@ +--- + +prometheus_node_exporter_install: false +filebeat_install: false +metricbeat_install: false + +beats_config: + filebeat: +# output_logstash_hosts: ["localhost:5044"] # The destination hosts for filebeat-gathered logs +# extra_logs_paths: # The array is optional, if you need to add more paths or files to scrape for logs +# - /var/log/myapp/*.log + metricbeat: +# output_logstash_hosts: ["localhost:5044"] # The destination hosts for metricbeat-gathered metrics +# diskio: # Diskio retrieves metrics for all disks partitions by default. When diskio.include_devices is defined, only look for defined partitions +# include_devices: ["sda", "sdb", "nvme0n1", "nvme1n1", "nvme2n1"] + + +cluster_vars: + dns_nameserver_zone: &dns_nameserver_zone "" # The zone that dns_server will operate on. gcloud dns needs a trailing '.'. Leave blank if no external DNS (use IPs only) + dns_user_domain: "{%- if _dns_nameserver_zone -%}{{cloud_type}}-{{region}}.{{app_class}}.{{buildenv}}.{{_dns_nameserver_zone}}{%- endif -%}" # A user-defined _domain_ part of the FDQN, (if more prefixes are required before the dns_nameserver_zone) + instance_profile_name: "" + custom_tagslabels: + inv_resident_id: "myresident" + inv_proposition_id: "myproposition" + inv_cost_centre: "0000000000" +_dns_nameserver_zone: *dns_nameserver_zone diff --git a/EXAMPLE/cluster_defs/azure/testid/westeurope/cluster_vars__region.yml b/EXAMPLE/cluster_defs/azure/testid/westeurope/cluster_vars__region.yml new file mode 100644 index 00000000..36d67fba --- /dev/null +++ b/EXAMPLE/cluster_defs/azure/testid/westeurope/cluster_vars__region.yml @@ -0,0 +1,9 @@ +--- + +_ubuntu2004image: { "publisher": "canonical", "offer": "0001-com-ubuntu-server-focal", "sku": "20_04-lts-gen2", "version": "latest" } +_ubuntu1804image: { "publisher": "canonical", "offer": "UbuntuServer", "sku": "18_04-lts-gen2", "version": "latest" } +_centos7image: { "publisher": "OpenLogic", "offer": "CentOS", "sku": "7_9-gen2", "version": "latest" } + +cluster_vars: + image: "{{_ubuntu2004image}}" + diff --git a/EXAMPLE/cluster_defs/azure/testid/westeurope/sandbox/cluster_vars__buildenv.yml b/EXAMPLE/cluster_defs/azure/testid/westeurope/sandbox/cluster_vars__buildenv.yml new file mode 100644 index 00000000..9431e802 --- /dev/null +++ b/EXAMPLE/cluster_defs/azure/testid/westeurope/sandbox/cluster_vars__buildenv.yml @@ -0,0 +1,69 @@ +--- + +cluster_vars: + sandbox: + azure_subscription_id: !vault | + $ANSIBLE_VAULT;1.2;AES256;sandbox + 7669080460651349243347331538721104778691266429457726036813912140404310 + azure_client_id: !vault | + $ANSIBLE_VAULT;1.2;AES256;sandbox + 7669080460651349243347331538721104778691266429457726036813912140404310 + azure_secret: !vault | + $ANSIBLE_VAULT;1.2;AES256;sandbox + 7669080460651349243347331538721104778691266429457726036813912140404310 + azure_tenant: !vault | + $ANSIBLE_VAULT;1.2;AES256;sandbox + 7669080460651349243347331538721104778691266429457726036813912140404310 + ssh_connection_cfg: + host: &host_ssh_connection_cfg + ansible_user: "ansible" + ansible_ssh_private_key_file: !vault | + $ANSIBLE_VAULT;1.2;AES256;sandbox + 7669080460651349243347331538721104778691266429457726036813912140404310 + bastion: + ssh_args: '-o ProxyCommand="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ./id_rsa_bastion -W %h:%p -q user@192.168.0.1"' + ssh_priv_key: !vault | + $ANSIBLE_VAULT;1.2;AES256;sandbox + 7669080460651349243347331538721104778691266429457726036813912140404310 + azure_resource_group: "compute" + vnet_name: "{{buildenv}}" + vpc_subnet_name_prefix: "{{buildenv}}-test-{{region}}" +# nsupdate_cfg: {server: "", key_name: "", key_secret: ""} # If you're using bind9 (or other nsupdate-compatible 'dns_server') + + hosttype_vars: + sys: + auto_volumes: [ ] + flavor: Standard_B1ls + version: "{{sys_version | default('')}}" + vms_by_az: { 1: 1, 2: 0, 3: 0 } + +# sysdisks2: +# auto_volumes: +# - { device_name: "0", disk_size_gb: 1, storage_account_type: "StandardSSD_LRS", mountpoint: "/media/mysvc0", fstype: "ext4", caching: "ReadOnly", perms: { owner: "root", group: "root", mode: "775" } } +# - { device_name: "1", disk_size_gb: 1, storage_account_type: "StandardSSD_LRS", mountpoint: "/media/mysvc1", fstype: "ext4", caching: "ReadOnly" } +# flavor: Standard_B1ls +# os_disk_size_gb: "35" # This is optional, and if set, MUST be bigger than the original image size (e.g. 30GB for Ubuntu2004) +# version: "{{sysdisks_version | default('')}}" +# vms_by_az: { 1: 1, 2: 0, 3: 0 } + +# sysdisks2lvm: +# auto_volumes: +# - { device_name: "0", disk_size_gb: 1, storage_account_type: "StandardSSD_LRS", mountpoint: "/media/mysvc0", fstype: "ext4", caching: "ReadOnly" } +# - { device_name: "1", disk_size_gb: 1, storage_account_type: "StandardSSD_LRS", mountpoint: "/media/mysvc0", fstype: "ext4", caching: "ReadOnly" } +# lvmparams: {vg_name: vg0, lv_name: lv0, lv_size: +100%FREE} +# flavor: Standard_B1ls +# os_disk_size_gb: "35" # This is optional, and if set, MUST be bigger than the original image size (e.g. 30GB for Ubuntu2004) +# version: "{{sysdisks_version | default('')}}" +# vms_by_az: { 1: 1, 2: 0, 3: 0 } + +# sysdisks4: +# auto_volumes: +# - { device_name: "3", disk_size_gb: 1, storage_account_type: "StandardSSD_LRS", mountpoint: "/media/mysvc3", fstype: "ext4", caching: "ReadOnly" } +# - { device_name: "1", disk_size_gb: 1, storage_account_type: "StandardSSD_LRS", mountpoint: "/media/mysvc1", fstype: "ext4", caching: "ReadOnly" } +# - { device_name: "0", disk_size_gb: 1, storage_account_type: "StandardSSD_LRS", mountpoint: "/media/mysvc0", fstype: "ext4", caching: "ReadOnly" } +# - { device_name: "2", disk_size_gb: 1, storage_account_type: "StandardSSD_LRS", mountpoint: "/media/mysvc2", fstype: "ext4", caching: "ReadOnly" } +# flavor: Standard_B2s # B1ls only supports 2 disks (B2s supports 4) +# version: "{{sysdisks_version | default('')}}" +# vms_by_az: { 1: 1, 2: 1, 3: 0 } + +_host_ssh_connection_cfg: { <<: *host_ssh_connection_cfg } diff --git a/EXAMPLE/cluster_defs/esxifree/cluster_vars__cloud.yml b/EXAMPLE/cluster_defs/esxifree/cluster_vars__cloud.yml new file mode 100644 index 00000000..da086063 --- /dev/null +++ b/EXAMPLE/cluster_defs/esxifree/cluster_vars__cloud.yml @@ -0,0 +1,9 @@ +--- + +_scheme_rmvm_keepdisk_rollback__copy_or_move: "move" + +cluster_vars: + dns_cloud_internal_domain: "" # The cloud-internal zone as defined by the cloud provider (e.g. GCP, AWS) + dns_server: "" # Specify DNS server. nsupdate, route53 or clouddns. If empty string is specified, no DNS will be added. + inventory_ip: "private" # 'public' or 'private', (private in case we're operating in a private LAN). If public, 'assign_public_ip' must be 'yes' + hardware_version: "19" diff --git a/EXAMPLE/cluster_defs/esxifree/testid/cluster_vars__clusterid.yml b/EXAMPLE/cluster_defs/esxifree/testid/cluster_vars__clusterid.yml new file mode 100644 index 00000000..79499d24 --- /dev/null +++ b/EXAMPLE/cluster_defs/esxifree/testid/cluster_vars__clusterid.yml @@ -0,0 +1,25 @@ +--- + +prometheus_node_exporter_install: false +filebeat_install: false +metricbeat_install: false + +beats_config: + filebeat: +# output_logstash_hosts: ["localhost:5044"] # The destination hosts for filebeat-gathered logs +# extra_logs_paths: # The array is optional, if you need to add more paths or files to scrape for logs +# - /var/log/myapp/*.log + metricbeat: +# output_logstash_hosts: ["localhost:5044"] # The destination hosts for metricbeat-gathered metrics +# diskio: # Diskio retrieves metrics for all disks partitions by default. When diskio.include_devices is defined, only look for defined partitions +# include_devices: ["sda", "sdb", "nvme0n1", "nvme1n1", "nvme2n1"] + + +cluster_vars: + dns_nameserver_zone: &dns_nameserver_zone "" # The zone that dns_server will operate on. gcloud dns needs a trailing '.'. Leave blank if no external DNS (use IPs only) + dns_user_domain: "{%- if _dns_nameserver_zone -%}{{cloud_type}}-{{region}}.{{app_class}}.{{buildenv}}.{{_dns_nameserver_zone}}{%- endif -%}" # A user-defined _domain_ part of the FDQN, (if more prefixes are required before the dns_nameserver_zone) + custom_tagslabels: + inv_resident_id: "myresident" + inv_proposition_id: "myproposition" + inv_cost_centre: "0000000000" +_dns_nameserver_zone: *dns_nameserver_zone diff --git a/EXAMPLE/cluster_defs/esxifree/testid/homelab/cluster_vars__region.yml b/EXAMPLE/cluster_defs/esxifree/testid/homelab/cluster_vars__region.yml new file mode 100644 index 00000000..6873eb31 --- /dev/null +++ b/EXAMPLE/cluster_defs/esxifree/testid/homelab/cluster_vars__region.yml @@ -0,0 +1,13 @@ +--- + +_ubuntu2004image: "gold-ubuntu2004l-20210415101808" +_centos7image: "gold-ubuntu2004l-20210415101808" + +cluster_vars: + image: "{{_ubuntu2004image}}" + esxi_ip: "192.168.1.3" + username: "svc" + password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 7669080460651349243347331538721104778691266429457726036813912140404310 + datastore: "4tb-evo860-ssd" diff --git a/EXAMPLE/cluster_defs/esxifree/testid/homelab/sandbox/cluster_vars__buildenv.yml b/EXAMPLE/cluster_defs/esxifree/testid/homelab/sandbox/cluster_vars__buildenv.yml new file mode 100644 index 00000000..169da345 --- /dev/null +++ b/EXAMPLE/cluster_defs/esxifree/testid/homelab/sandbox/cluster_vars__buildenv.yml @@ -0,0 +1,37 @@ +--- + +cluster_vars: + sandbox: + ssh_connection_cfg: + host: &host_ssh_connection_cfg + ansible_user: "ansible" + ansible_ssh_private_key_file: !vault | + $ANSIBLE_VAULT;1.2;AES256;sandbox + 7669080460651349243347331538721104778691266429457726036813912140404310 +# bastion: +# ssh_args: '-o ProxyCommand="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ./id_rsa_bastion -W %h:%p -q user@192.168.0.1"' +# ssh_priv_key: !vault | +# $ANSIBLE_VAULT;1.2;AES256;sandbox +# 7669080460651349243347331538721104778691266429457726036813912140404310 + networks: + - networkName: "VM Network" + virtualDev: vmxnet3 + cloudinit_netplan: { ethernets: { eth0: { dhcp4: true } } } +# nsupdate_cfg: {server: "", key_name: "", key_secret: ""} # If you're using bind9 (or other nsupdate-compatible 'dns_server') + + hosttype_vars: + sys: + auto_volumes: [ ] + flavor: { num_cpus: "2", memory_mb: "2048" } + version: "{{sys_version | default('')}}" + vms_by_az: { a: 1, b: 1, c: 0 } + + sysdisks2: + auto_volumes: + - { mountpoint: "/media/mysvc1", volume_size: 1, provisioning_type: "thin", fstype: "ext4" } + - { mountpoint: "/media/mysvc2", volume_size: 1, provisioning_type: "thin", fstype: "ext4" } + flavor: { num_cpus: "2", memory_mb: "2048" } + version: "{{sys_version | default('')}}" + vms_by_az: { a: 1, b: 1, c: 0 } + +_host_ssh_connection_cfg: { <<: *host_ssh_connection_cfg } diff --git a/EXAMPLE/jenkinsfiles/Jenkinsfile_ops b/EXAMPLE/jenkinsfiles/Jenkinsfile_ops index 11a6d2ac..1f06f7c6 100644 --- a/EXAMPLE/jenkinsfiles/Jenkinsfile_ops +++ b/EXAMPLE/jenkinsfiles/Jenkinsfile_ops @@ -1,10 +1,10 @@ #!groovy //These will not be needed if we're running this as a pipeline SCM job, as these are automatically added to the 'scm' variable, but if we instead just cut & paste this file into a pipeline job, they will be used as fallback -def DEFAULT_CLUSTERVERSE_URL = "https://github.com/sky-uk/clusterverse" -def DEFAULT_CLUSTERVERSE_BRANCH = "master" +def DEFAULT_CLUSTERVERSE_URL = "https://github.com/dseeley/clusterverse" +def DEFAULT_CLUSTERVERSE_BRANCH = "dps_esxi" -def DEFAULT_CLUSTERVERSE_TESTSUITE_URL = "https://github.com/sky-uk/clusterverse-test" +def DEFAULT_CLUSTERVERSE_TESTSUITE_URL = "https://github.com/dseeley/clusterverse_test" def DEFAULT_CLUSTERVERSE_TESTSUITE_BRANCH = "master" //This allows us to create our own Docker image for this specific use-case. Once it is built, it will not be rebuilt, so only adds delay the first time we use it. @@ -47,8 +47,8 @@ properties([ parameters([ string(name: 'APP_NAME', description: "An optional custom app_name to override the default in the playbook"), booleanParam(name: 'APPEND_BUILD_NUMBER', defaultValue: false, description: 'Tick the box to append the Jenkins BUILD_NUMBER to APP_NAME'), - choice(name: 'CLOUD_REGION', choices: ['aws/us-west-2', 'aws/eu-central-1', 'aws/eu-west-1', 'gcp/us-west2', 'gcp/europe-west1'], description: "Choose a cloud/region"), - choice(name: 'BUILDENV', choices: ['sandbox', 'dev', 'mgmt', 'tools', 'stage', 'prod'], description: "Choose an environment to deploy"), + choice(name: 'CLOUD_REGION', choices: ['esxifree/dougalab', 'aws/eu-west-1', 'gcp/europe-west1'], description: "Choose a cloud/region"), + choice(name: 'BUILDENV', choices: ['sandbox', 'dev', 'stage', 'prod'], description: "Choose an environment to deploy"), string(name: 'CLUSTER_ID', defaultValue: '', description: "Select a cluster_id to deploy", trim: true), booleanParam(name: 'DNS_FORCE_DISABLE', defaultValue: false, description: 'Tick the box to force disable the DNS as defined in playbook'), choice(name: 'DEPLOY_TYPE', choices: ['deploy', 'redeploy', 'clean'], description: "Choose the deploy type"), diff --git a/README.md b/README.md index 99f81a75..7dc45124 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # clusterverse   [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) ![PRs Welcome](https://img.shields.io/badge/PRs-Welcome-brightgreen.svg) A full-lifecycle, immutable cloud infrastructure cluster management **role**, using Ansible. -+ **Multi-cloud:** clusterverse can manage cluster lifecycle in AWS and GCP ++ **Multi-cloud:** clusterverse can manage cluster lifecycle in AWS, GCP, Free ESXi (standalone host only, not vCentre) and Azure + **Deploy:** You define your infrastructure as code (in Ansible yaml), and clusterverse will deploy it + **Scale-up:** If you change the cluster definitions and rerun the deploy, new nodes will be added. + **Redeploy (e.g. up-version):** If you need to up-version, or replace the underlying OS, (i.e. to achieve fully immutable, zero-patching redeploys), the `redeploy.yml` playbook will replace each node in the cluster (via various redeploy schemes), and rollback if any failures occur. @@ -38,6 +38,22 @@ To active the pipenv: + During execution, the json file will be copied locally because the Ansible GCP modules often require the file as input. + Google Cloud SDK needs to be installed to run gcloud command-line (e.g. to disable delete protection) - this is handled by `pipenv install` +### ESXi (free) ++ Username & password for a privileged user on an ESXi host ++ SSH must be enabled on the host ++ Set the `Config.HostAgent.vmacore.soap.maxSessionCount` variable to 0 to allow many concurrent tests to run. + +### Azure ++ Create an Azure account. ++ Create a Tenant and a Subscription ++ Create a Resource group and networks/subnetworks within that. ++ Create a service principal - add the credentials to: + + `cluster_vars[buildenv].azure_subscription_id` + + `cluster_vars[buildenv].azure_client_id` + + `cluster_vars[buildenv].azure_secret` + + `cluster_vars[buildenv].azure_tenant` + + ### DNS DNS is optional. If unset, no DNS names will be created. If DNS is required, you will need a DNS zone delegated to one of the following: + nsupdate (e.g. bind9) @@ -227,3 +243,4 @@ The role is designed to run in two modes: + If `canary=start`, only the first node is redeployed. If `canary=finish`, only the remaining (non-first), nodes are replaced. If `canary=none`, all nodes are redeployed. + If the process fails for any reason, the old VMs are reinstated (and the disks reattached to the old nodes), and the new VMs are stopped (rollback) + To delete the old VMs, either set '-e canary_tidy_on_success=true', or call redeploy.yml with '-e canary=tidy' + + (Azure functionality coming soon) diff --git a/_dependencies/action_plugins/ansible_vault.py b/_dependencies/action_plugins/ansible_vault.py new file mode 100644 index 00000000..65e4cf4f --- /dev/null +++ b/_dependencies/action_plugins/ansible_vault.py @@ -0,0 +1,98 @@ +# Copyright 2020 Dougal Seeley +# BSD 3-Clause License + +from __future__ import (absolute_import, division, print_function) + +__metaclass__ = type + +from ansible import constants as C +from ansible.plugins.action import ActionBase +from ansible.parsing.vault import VaultLib, VaultSecret, parse_vaulttext_envelope, parse_vaulttext +from ansible.utils.display import Display + +display = Display() + +################################# +# An action plugin to perform vault encrypt/decrypt operations inside a playbook. Can use either user-provided id/pass, or can use already-loaded vault secrets. +################################# +# +# - name: Encrypt using user-provided vaultid and vaultpass +# ansible_vault: +# vaultid: sandbox +# vaultpass: asdf +# plaintext: "sometext" +# action: encrypt +# register: r__ansible_vault_encrypt +# - debug: msg={{r__ansible_vault_encrypt}} +# +# - name: Decrypt using user-provided vaultid and vaultpass +# ansible_vault: +# vaultid: sandbox +# vaultpass: asdf +# vaulttext: "$ANSIBLE_VAULT;1.2;AES256;sandbox\n303562383536366435346466313764636533353438653463373765616365623130333633613139326235633064643338316665653531663030643139373131390a323233356239303864343336663238616535386638646566623036383130643638373465646331316664636564376161376137623432616561343631313262620a3561656131353364616136373866343963626561366236653538633734653165" +# action: decrypt +# register: r__ansible_vault_decrypt +# - debug: msg={{r__ansible_vault_decrypt}} +# +# - name: Encrypt using already-loaded vault secrets (from command-line, ansible.cfg etc) +# ansible_vault: +# plaintext: "sometext" +# action: encrypt +# register: r__ansible_vault_encrypt +# - debug: msg={{r__ansible_vault_encrypt}} +# +# - name: Decrypt using already-loaded vault secrets (from command-line, ansible.cfg etc) +# ansible_vault: +# vaulttext: "$ANSIBLE_VAULT;1.2;AES256;sandbox\n303562383536366435346466313764636533353438653463373765616365623130333633613139326235633064643338316665653531663030643139373131390a323233356239303864343336663238616535386638646566623036383130643638373465646331316664636564376161376137623432616561343631313262620a3561656131353364616136373866343963626561366236653538633734653165" +# action: decrypt +# register: r__ansible_vault_decrypt +# - debug: msg={{r__ansible_vault_decrypt}} +# +################################# + +class ActionModule(ActionBase): + TRANSFERS_FILES = False + + def run(self, tmp=None, task_vars=None): + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp is deprecated + + # If user supplies vault-id and vault-pass, use them. Otherwise use those that are automatically loaded with the playbook + if 'vaultpass' in self._task.args: + oVaultSecret = VaultSecret(self._task.args["vaultpass"].encode('utf-8')) + if 'vaultid' in self._task.args: + oVaultLib = VaultLib([(self._task.args["vaultid"], oVaultSecret)]) + else: + display.v(u'No vault-id supplied, using default identity.') + oVaultLib = VaultLib([(C.DEFAULT_VAULT_IDENTITY, oVaultSecret)]) + else: + display.v(u'No vault-id or vault-pass supplied, using playbook-sourced variables.') + oVaultLib = self._loader._vault + if len(self._loader._vault.secrets) == 0: + display.warning("No Vault secrets loaded by config and none supplied to plugin. Vault operations are not possible.") + + if self._task.args["action"] == "encrypt": + if "plaintext" not in self._task.args: + return {"failed": True, "msg": "'plaintext' is required for encrypt."} + + b_vaulttext = oVaultLib.encrypt(self._task.args["plaintext"]) + b_ciphertext, b_version, cipher_name, vault_id = parse_vaulttext_envelope(b_vaulttext) + + vaulttext_header = b_vaulttext.decode('utf-8').split('\n',1)[0] + result['vaulttext'] = vaulttext_header + "\n" + b_ciphertext.decode('utf-8') + result['plaintext'] = self._task.args["plaintext"] + + else: + if "vaulttext" not in self._task.args: + return {"failed": True, "msg": "'vaulttext' is required for decrypt."} + + plaintext = oVaultLib.decrypt(self._task.args["vaulttext"]) + result['vaulttext'] = self._task.args["vaulttext"] + result['plaintext'] = plaintext + + result['failed'] = False + + return result diff --git a/_dependencies/library/esxifree_guest.py b/_dependencies/library/esxifree_guest.py new file mode 100644 index 00000000..8a7e736f --- /dev/null +++ b/_dependencies/library/esxifree_guest.py @@ -0,0 +1,1133 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright (c) 2021, Dougal Seeley +# https://github.com/dseeley/esxifree_guest +# BSD 3-Clause License + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ['preview'], 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: esxifree_guest +short_description: Manages virtual machines in ESXi without a dependency on the vSphere/ vCenter API. +description: > + This module can be used to create new virtual machines from scratch or from templates or other virtual machines (i.e. clone them), + delete, or manage the power state of virtual machine such as power on, power off, suspend, shutdown, reboot, restart etc., +version_added: '2.7' +author: +- Dougal Seeley (ansible@dougalseeley.com) +requirements: +- python >= 2.7 +- paramiko +- xmltodict +notes: + - Please make sure that the user used for esxifree_guest should have correct level of privileges. + - Tested on vSphere 7.0.2 +options: + hostname: + description: + - The hostname or IP address of the ESXi server. + required: true + type: str + username: + description: + - The username to access the ESXi server at C(hostname). + required: true + type: str + password: + description: + - The password of C(username) for the ESXi server, or the password for the private key (if required). + required: true + type: str + state: + description: + - Specify the state the virtual machine should be in. + - 'If C(state) is set to C(present) and virtual machine exists, ensure the virtual machine + configurations conforms to task arguments.' + - 'If C(state) is set to C(absent) and virtual machine exists, then the specified virtual machine + is removed with its associated components.' + - 'If C(state) is set to one of the following C(poweredon), C(poweredoff), C(present) + and virtual machine does not exists, then virtual machine is deployed with given parameters.' + - 'If C(state) is set to C(poweredon) and virtual machine exists with powerstate other than powered on, + then the specified virtual machine is powered on.' + - 'If C(state) is set to C(poweredoff) and virtual machine exists with powerstate other than powered off, + then the specified virtual machine is powered off.' + - 'If C(state) is set to C(shutdownguest) and virtual machine exists, then the virtual machine is shutdown.' + - 'If C(state) is set to C(rebootguest) and virtual machine exists, then the virtual machine is rebooted.' + - 'If C(state) is set to C(unchanged) the state of the VM will not change (if it's on/off, it will stay so). Used for updating annotations.' + choices: [ present, absent, poweredon, poweredoff, shutdownguest, rebootguest, unchanged ] + default: present + name: + description: + - Name of the virtual machine to work with. + - Virtual machine names in ESXi are unique + - This parameter is required, if C(state) is set to C(present) and virtual machine does not exists. + - This parameter is case sensitive. + type: str + moid: + description: + - Managed Object ID of the virtual machine to manage + - This is required if C(name) is not supplied. + - If virtual machine does not exists, then this parameter is ignored. + - Will be ignored on virtual machine creation + type: str + template: + description: + - Template or existing virtual machine used to create new virtual machine. + - If this value is not set, virtual machine is created without using a template. + - If the virtual machine already exists, this parameter will be ignored. + - This parameter is case sensitive. + type: str + hardware: + description: + - Manage virtual machine's hardware attributes. + type: dict + suboptions: + version: + description: + - The Virtual machine hardware version. Default is 15 (ESXi 6.7U2 and onwards). + type: int + default: 15 + required: false + num_cpus: + description: + - Number of CPUs. + - C(num_cpus) must be a multiple of C(num_cpu_cores_per_socket). + type: int + default: 2 + required: false + num_cpu_cores_per_socket: + description: + - Number of Cores Per Socket. + type: int + default: 1 + required: false + hotadd_cpu: + description: + - Allow virtual CPUs to be added while the virtual machine is running. + type: bool + required: false + memory_mb: + description: + - Amount of memory in MB. + type: int + default: 2048 + required: false + memory_reservation_lock: + description: + - If set true, memory resource reservation for the virtual machine + will always be equal to the virtual machine's memory size. + type: bool + required: false + hotadd_memory: + description: + - Allow memory to be added while the virtual machine is running. + type: bool + required: false + guest_id: + description: + - Set the guest ID. + - This parameter is case sensitive. + - 'Examples:' + - " virtual machine with RHEL7 64 bit, will be 'rhel7-64'" + - " virtual machine with CentOS 7 (64-bit), will be 'centos7-64'" + - " virtual machine with Debian 9 (Stretch) 64 bit, will be 'debian9-64'" + - " virtual machine with Ubuntu 64 bit, will be 'ubuntu-64'" + - " virtual machine with Windows 10 (64 bit), will be 'windows9-64'" + - " virtual machine with Other (64 bit), will be 'other-64'" + - This field is required when creating a virtual machine, not required when creating from the template. + type: str + default: ubuntu-64 + disks: + description: + - A list of disks to add (or create via cloning). + - Resizing disks is not supported. + - Removing existing disks of the virtual machine is not supported. + required: false + type: list + suboptions: + boot: + description: + - Indicates that this is a boot disk. + required: false + default: no + type: bool + size_gb: + description: Specifies the size of the disk in base-2 GB. + type: int + required: true + type: + description: + - Type of disk provisioning + choices: [thin, thick, eagerzeroedthick] + type: str + required: false + default: thin + volname: + description: + - Volume name. This will be a suffix of the vmdk file, e.g. "testdisk" on a VM named "mynewvm", would yield mynewvm--testdisk.vmdk + type: str + required: true + src: + description: + - The source disk from which to create this disk. + required: false + type: dict + suboptions: + backing_filename: + description: + - The source file, e.g. "[datastore1] linux_dev/linux_dev--webdata.vmdk" + type: str + copy_or_move + description: + - Whether to copy (clone) from the source datastore, or move the file. Move will fail if source and destination datastore differ. + choices: [copy, move] + + cdrom: + description: + - A CD-ROM configuration for the virtual machine. + - 'Valid attributes are:' + - ' - C(type) (string): The type of CD-ROM, valid options are C(none), C(client) or C(iso). With C(none) the CD-ROM will be disconnected but present.' + - ' - C(iso_path) (string): The datastore path to the ISO file to use, in the form of C([datastore1] path/to/file.iso). Required if type is set C(iso).' + wait: + description: + - On creation, wait for the instance to obtain its IP address before returning. + type: bool + required: false + default: true + wait_timeout: + description: + - How long before wait gives up, in seconds. + type: int + required: false + default: 180 + force: + description: + - Delete the existing host if it exists. Use with extreme care! + type: bool + required: false + default: false + customvalues: + description: + - Define a list of custom values to set on virtual machine. + - A custom value object takes two fields C(key) and C(value). + - Incorrect key and values will be ignored. + version_added: '2.3' + cloudinit_userdata: + description: + - A list of userdata (per user) as defined U(https://cloudinit.readthedocs.io/en/latest/topics/examples.html). The + VM must already have cloud-init-vmware-guestinfo installed U(https://github.com/vmware/cloud-init-vmware-guestinfo) + networks: + description: + - A list of networks (in the order of the NICs). + - Removing NICs is not allowed, while reconfiguring the virtual machine. + - All parameters and VMware object names are case sensetive. + - 'One of the below parameters is required per entry:' + - ' - C(networkName) (string): Name of the portgroup for this interface. + - ' - C(virtualDev) (string): Virtual network device (one of C(e1000e), C(vmxnet3) (default), C(sriov)).' + - 'Optional parameters per entry (used for OS customization):' + - ' - C(cloudinit_ethernets) (dict): A list of C(ethernets) within the definition of C(Networking Config Version 2) + defined in U(https://cloudinit.readthedocs.io/en/latest/topics/network-config-format-v2.html)'. The + VM must already have cloud-init-vmware-guestinfo installed U(https://github.com/vmware/cloud-init-vmware-guestinfo) + datastore: + description: + - Specify datastore or datastore cluster to provision virtual machine. + type: str + required: true + +''' +EXAMPLES = r''' +- name: Create a virtual machine + esxifree_guest: + hostname: "192.168.1.3" + username: "svc" + password: "my_passsword" + datastore: "datastore1" + name: "test_asdf" + state: present + guest_id: ubuntu-64 + hardware: {"version": "15", "num_cpus": "2", "memory_mb": "2048"} + cloudinit_userdata: + - name: dougal + primary_group: dougal + sudo: "ALL=(ALL) NOPASSWD:ALL" + groups: "admin" + home: "/media/filestore/home/dougal" + ssh_import_id: None + lock_passwd: false + passwd: $6$j212wezy$7...YPYb2F + ssh_authorized_keys: ['ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACA+.................GIMhdojtl6mzVn38vXMzSL29LQ== ansible@dougalseeley.com'] + disks: + - {"boot": true, "size_gb": 16, "type": "thin"} + - {"size_gb": 2, "type": "thin", "volname": "test_new"} + - {"size_gb": 1, "type": "thin", "volname": "test_clone", "src": {"backing_filename": "[datastore1] linux_dev/linux_dev--webdata.vmdk", "copy_or_move": "copy"}}], + cdrom: {"type": "iso", "iso_path": "/vmfs/volumes/4tb-evo860-ssd/ISOs/ubuntu-18.04.4-server-amd64.iso"}, + networks: + - networkName: VM Network + virtualDev: vmxnet3 + cloudinit_ethernets: + eth0: + addresses: ["192.168.1.8/25"] + dhcp4: false + gateway4: 192.168.1.1 + nameservers: + addresses: ["192.168.1.2", "8.8.8.8", "8.8.4.4"] + search: ["local.dougalseeley.com"] + delegate_to: localhost + +- name: Clone a virtual machine + esxifree_guest: + hostname: "192.168.1.3" + username: "svc" + password: "my_passsword" + datastore: "datastore1" + template: "ubuntu1804-packer-template" + name: "test_asdf" + state: present + guest_id: ubuntu-64 + hardware: {"version": "15", "num_cpus": "2", "memory_mb": "2048"} + cloudinit_userdata: + - default + - name: dougal + primary_group: dougal + sudo: "ALL=(ALL) NOPASSWD:ALL" + groups: "admin" + home: "/media/filestore/home/dougal" + ssh_import_id: None + lock_passwd: true + ssh_authorized_keys: ['ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACA+.................GIMhdojtl6mzVn38vXMzSL29LQ== ansible@dougalseeley.com'] + disks: + - {"size_gb": 2, "type": "thin", "volname": "test_new"} + - {"size_gb": 1, "type": "thin", "volname": "test_clone", "src": {"backing_filename": "[datastore1] linux_dev/linux_dev--webdata.vmdk", "copy_or_move": "copy"}}], + networks: + - networkName: VM Network + virtualDev: vmxnet3 + cloudinit_ethernets: + eth0: + addresses: ["192.168.1.8/25"] + dhcp4: false + gateway4: 192.168.1.1 + nameservers: + addresses: ["192.168.1.2", "8.8.8.8", "8.8.4.4"] + search: ["local.dougalseeley.com"] + delegate_to: localhost + +- name: Delete a virtual machine + esxifree_guest: + hostname: "{{ esxi_ip }}" + username: "{{ username }}" + password: "{{ password }}" + name: test_vm_0001 + state: absent + delegate_to: localhost +''' + +RETURN = r''' +instance: + description: metadata about the new virtual machine + returned: always + type: dict + sample: None +''' + +import os +import time +import re +import json +import socket +import collections +import paramiko +import sys +import base64 +import yaml +import errno # For the python2.7 IOError, because FileNotFound is for python3 +import xmltodict + +# define a custom yaml representer to force quoted strings +yaml.add_representer(str, lambda dumper, data: dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"')) + +# For the soap client +try: + from urllib.request import Request, build_opener, HTTPSHandler, HTTPCookieProcessor + from urllib.response import addinfourl + from urllib.error import HTTPError + from http.cookiejar import CookieJar + from http.client import HTTPResponse +except ImportError: + from urllib2 import Request, build_opener, HTTPError, HTTPSHandler, HTTPCookieProcessor, addinfourl + from cookielib import CookieJar + from httplib import HTTPResponse +import ssl + +if sys.version_info[0] < 3: + from io import BytesIO as StringIO +else: + from io import StringIO + +# paramiko.util.log_to_file("paramiko.log") +# paramiko.common.logging.basicConfig(level=paramiko.common.DEBUG) + +try: + from ansible.module_utils.basic import AnsibleModule +except: + # For testing without Ansible (e.g on Windows) + class cDummyAnsibleModule(): + def __init__(self): + self.params={} + def exit_json(self, changed, **kwargs): + print(changed, json.dumps(kwargs, sort_keys=True, indent=4, separators=(',', ': '))) + def fail_json(self, msg): + print("Failed: " + msg) + exit(1) + +# Executes soap requests on the remote host. +class vmw_soap_client(object): + def __init__(self, host, username, password): + self.vmware_soap_session_cookie = None + self.host = host + response, cookies = self.send_req("<_this>ServiceInstance") + xmltodictresponse = xmltodict.parse(response.read()) + sessionManager_name = xmltodictresponse['soapenv:Envelope']['soapenv:Body']['RetrieveServiceContentResponse']['returnval']['sessionManager']['#text'] + + response, cookies = self.send_req("<_this>" + sessionManager_name + "" + username + "" + password + "") + self.vmware_soap_session_cookie = cookies['vmware_soap_session'].value + + def send_req(self, envelope_body=None): + envelope = '' + '' + str(envelope_body) + '' + cj = CookieJar() + req = Request( + url='https://' + self.host + '/sdk/vimService.wsdl', data=envelope.encode(), + headers={"Content-Type": "text/xml", "SOAPAction": "urn:vim25/6.7.3", "Accept": "*/*", "Cookie": "vmware_client=VMware; vmware_soap_session=" + str(self.vmware_soap_session_cookie)}) + + opener = build_opener(HTTPSHandler(context=ssl._create_unverified_context()), HTTPCookieProcessor(cj)) + num_send_attempts = 3 + for send_attempt in range(num_send_attempts): + try: + response = opener.open(req, timeout=30) + except HTTPError as err: + response = str(err) + except: + if send_attempt < num_send_attempts - 1: + time.sleep(1) + continue + else: + raise + break + + cookies = {i.name: i for i in list(cj)} + return (response[0] if isinstance(response, list) else response, cookies) # If the cookiejar contained anything, we get a list of two responses + + def wait_for_task(self, task, timeout=30): + time_s = int(timeout) + while time_s > 0: + response, cookies = self.send_req('<_this type="PropertyCollector">ha-property-collectorTaskfalseinfo' + task + 'false') + if isinstance(response, HTTPResponse) or isinstance(response, addinfourl): + xmltodictresponse = xmltodict.parse(response.read()) + if xmltodictresponse['soapenv:Envelope']['soapenv:Body']['RetrievePropertiesResponse']['returnval']['propSet']['val'] == 'running': + time.sleep(1) + time_s = time_s - 1 + elif xmltodictresponse['soapenv:Envelope']['soapenv:Body']['RetrievePropertiesResponse']['returnval']['propSet']['val']['state'] == 'success': + response = xmltodictresponse['soapenv:Envelope']['soapenv:Body']['RetrievePropertiesResponse']['returnval']['propSet']['val']['state'] + break + elif xmltodictresponse['soapenv:Envelope']['soapenv:Body']['RetrievePropertiesResponse']['returnval']['propSet']['val']['state'] == 'error': + response = str(xmltodictresponse) + break + else: + break + return response + + +# Executes a command on the remote host. +class SSHCmdExec(object): + def __init__(self, hostname, username=None, password=None, pkeyfile=None, pkeystr=None): + self.hostname = hostname + + try: + if pkeystr and pkeystr != "": + pkey_fromstr = paramiko.RSAKey.from_private_key(StringIO(pkeystr), password) + if pkeyfile and pkeyfile != "": + pkey_fromfile = paramiko.RSAKey.from_private_key_file(pkeyfile, password) + except paramiko.ssh_exception.PasswordRequiredException as auth_err: + print("Authentication failure, Password required" + "\n\n" + str(auth_err)) + exit(1) + except paramiko.ssh_exception.SSHException as auth_err: + print("Authentication failure, SSHException" + "\n\n" + str(auth_err)) + exit(1) + except: + print("Unexpected error: ", sys.exc_info()[0]) + raise + else: + if pkeystr: + self.pkey = pkey_fromstr + if pkeyfile: + if pkey_fromstr != pkey_fromfile: + print("Both private key file and private key string specified and not equal!") + exit(1) + elif pkeyfile: + self.pkey = pkey_fromfile + + # Create instance of SSHClient object + self.remote_conn_client = paramiko.SSHClient() + self.remote_conn_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + # initiate SSH connection + try: + if hasattr(self, 'pkey'): + self.remote_conn_client.connect(hostname=hostname, username=username, pkey=self.pkey, timeout=10, look_for_keys=False, allow_agent=False) + else: + self.remote_conn_client.connect(hostname=hostname, username=username, password=password, timeout=10, look_for_keys=False, allow_agent=False) + except socket.error as sock_err: + print("Connection timed-out to " + hostname) # + "\n\n" + str(sock_err) + exit(1) + except paramiko.ssh_exception.AuthenticationException as auth_err: + print("Authentication failure, unable to connect to " + hostname + " as " + username + "\n\n" + str(auth_err) + "\n\n" + str(sys.exc_info()[0])) # + str(auth_err)) + exit(1) + except: + print("Unexpected error: ", sys.exc_info()[0]) + raise + + # print("SSH connection established to " + hostname + " as " + username) + + def get_sftpClient(self): + return self.remote_conn_client.open_sftp() + + # execute the command and wait for it to finish + def exec_command(self, command_string): + # print("Command is: {0}".format(command_string)) + + (stdin, stdout, stderr) = self.remote_conn_client.exec_command(command_string) + if stdout.channel.recv_exit_status() != 0: # Blocking call + raise IOError(stderr.read()) + + return stdin, stdout, stderr + + +class esxiFreeScraper(object): + vmx_skeleton = collections.OrderedDict() + vmx_skeleton['.encoding'] = "UTF-8" + vmx_skeleton['config.version'] = "8" + vmx_skeleton['pciBridge0.present'] = "TRUE" + vmx_skeleton['svga.present'] = "TRUE" + vmx_skeleton['svga.autodetect'] = "TRUE" + vmx_skeleton['pciBridge4.present'] = "TRUE" + vmx_skeleton['pciBridge4.virtualDev'] = "pcieRootPort" + vmx_skeleton['pciBridge4.functions'] = "8" + vmx_skeleton['pciBridge5.present'] = "TRUE" + vmx_skeleton['pciBridge5.virtualDev'] = "pcieRootPort" + vmx_skeleton['pciBridge5.functions'] = "8" + vmx_skeleton['pciBridge6.present'] = "TRUE" + vmx_skeleton['pciBridge6.virtualDev'] = "pcieRootPort" + vmx_skeleton['pciBridge6.functions'] = "8" + vmx_skeleton['pciBridge7.present'] = "TRUE" + vmx_skeleton['pciBridge7.virtualDev'] = "pcieRootPort" + vmx_skeleton['pciBridge7.functions'] = "8" + vmx_skeleton['vmci0.present'] = "TRUE" + vmx_skeleton['hpet0.present'] = "TRUE" + vmx_skeleton['floppy0.present'] = "FALSE" + vmx_skeleton['usb.present'] = "TRUE" + vmx_skeleton['ehci.present'] = "TRUE" + vmx_skeleton['tools.syncTime'] = "TRUE" + vmx_skeleton['scsi0.virtualDev'] = "pvscsi" + vmx_skeleton['scsi0.present'] = "TRUE" + vmx_skeleton['disk.enableuuid'] = "TRUE" + + def __init__(self, hostname, username='root', password=None, name=None, moid=None): + self.soap_client = vmw_soap_client(host=hostname, username=username, password=password) + self.esxiCnx = SSHCmdExec(hostname=hostname, username=username, password=password) + self.name, self.moid = self.get_vm(name, moid) + if self.moid is None: + self.name = name + + def get_vm(self, name=None, moid=None): + response, cookies = self.soap_client.send_req('<_this type="PropertyCollector">ha-property-collectorVirtualMachinefalsenameha-folder-vmtraverseChildFolderchildEntity traverseChildDatacentervmFoldertraverseChild ') + xmltodictresponse = xmltodict.parse(response.read()) + allVms = [{'moid': a['obj']['#text'], 'name': a['propSet']['val']['#text']} for a in xmltodictresponse['soapenv:Envelope']['soapenv:Body']['RetrievePropertiesExResponse']['returnval']['objects']] + for vm in allVms: + if ((name and name == vm['name']) or (moid and moid == vm['moid'])): + return vm['name'], vm['moid'] + return None, None + + def get_vmx(self, moid): + (stdin, stdout, stderr) = self.esxiCnx.exec_command("vim-cmd vmsvc/get.filelayout " + str(moid) + " | grep 'vmPathName = ' | sed -r 's/^\s+vmPathName = \"(.*?)\",/\\1/g'") + vmxPathName = stdout.read().decode('UTF-8').lstrip("\r\n").rstrip(" \r\n") + vmxPath = re.sub(r"^\[(.*?)]\s+(.*?)$", r"/vmfs/volumes/\1/\2", vmxPathName) + + if vmxPath: + sftp_cnx = self.esxiCnx.get_sftpClient() + vmxFileDict = {} + for vmxline in sftp_cnx.file(vmxPath).readlines(): + vmxline_params = re.search('^(?P.*?)\s*=\s*(?P.*)$', vmxline) + if vmxline_params and vmxline_params.group('key') and vmxline_params.group('value'): + vmxFileDict[vmxline_params.group('key').strip(" \"\r\n").lower()] = vmxline_params.group('value').strip(" \"\r\n") + + return vmxPath, vmxFileDict + + def put_vmx(self, vmxDict, vmxPath): + # print(json.dumps(vmxDict, sort_keys=True, indent=4, separators=(',', ': '))) + vmxDict = collections.OrderedDict(sorted(vmxDict.items())) + vmxStr = StringIO() + for vmxKey, vmxVal in vmxDict.items(): + vmxStr.write(str(vmxKey.lower()) + " = " + "\"" + str(vmxVal) + "\"\n") + vmxStr.seek(0) + sftp_cnx = self.esxiCnx.get_sftpClient() + try: + sftp_cnx.stat(vmxPath) + sftp_cnx.remove(vmxPath) + except IOError as e: # python 2.7 + if e.errno == errno.ENOENT: + pass + except FileNotFoundError: # python 3.x + pass + sftp_cnx.putfo(vmxStr, vmxPath, file_size=0, callback=None, confirm=True) + + def create_vm(self, vmTemplate=None, annotation=None, datastore=None, hardware=None, guest_id=None, disks=None, cdrom=None, customvalues=None, networks=None, cloudinit_userdata=None): + vmPathDest = "/vmfs/volumes/" + datastore + "/" + self.name + + ## Sanity checks + for dryRunDisk in [newDisk for newDisk in disks if ('src' in newDisk and newDisk['src'] is not None)]: + if 'copy_or_move' not in dryRunDisk['src']: + return ("'copy_or_move' parameter is mandatory when src is specified for a disk.") + if 'backing_filename' not in dryRunDisk['src']: + return ("'backing_filename' parameter is mandatory when src is specified for a disk.") + + dryRunDiskFileInfo = re.search('^\[(?P.*?)\] *(?P.*\/(?P(?P.*?)(?:--(?P.*?))?\.vmdk))$', dryRunDisk['src']['backing_filename']) + try: + self.esxiCnx.exec_command("vmkfstools -g /vmfs/volumes/" + dryRunDiskFileInfo.group('datastore') + "/" + dryRunDiskFileInfo.group('fulldiskpath')) + except IOError as e: + return "'" + dryRunDisk['src']['backing_filename'] + "' is not accessible (is the VM turned on?)\n" + str(e) + + # Create VM directory + self.esxiCnx.exec_command("mkdir -p " + vmPathDest) + + vmxDict = collections.OrderedDict(esxiFreeScraper.vmx_skeleton) + + diskCount = 0 + + # First apply any vmx settings from the template. + # These will be overridden by explicit configuration. + if vmTemplate: + template_name, template_moid = self.get_vm(vmTemplate, None) + if template_moid: + template_vmxPath, template_vmxDict = self.get_vmx(template_moid) + + # Generic settings + vmxDict.update({"guestos": template_vmxDict['guestos']}) + + # Hardware settings + vmxDict.update({"virtualhw.version": template_vmxDict['virtualhw.version']}) + vmxDict.update({"memsize": template_vmxDict['memsize']}) + if 'numvcpus' in template_vmxDict: + vmxDict.update({"numvcpus": template_vmxDict['numvcpus']}) + if 'cpuid.coresPerSocket' in template_vmxDict: + vmxDict.update({"cpuid.coresPerSocket": template_vmxDict['cpuid.coresPerSocket']}) + if 'vcpu.hotadd' in template_vmxDict: + vmxDict.update({"vcpu.hotadd": template_vmxDict['vcpu.hotadd']}) + if 'mem.hotadd' in template_vmxDict: + vmxDict.update({"mem.hotadd": template_vmxDict['mem.hotadd']}) + if 'sched.mem.pin' in template_vmxDict: + vmxDict.update({"sched.mem.pin": template_vmxDict['sched.mem.pin']}) + + # Network settings + netCount = 0 + while "ethernet" + str(netCount) + ".virtualdev" in template_vmxDict: + vmxDict.update({"ethernet" + str(netCount) + ".virtualdev": template_vmxDict["ethernet" + str(netCount) + ".virtualdev"]}) + vmxDict.update({"ethernet" + str(netCount) + ".networkname": template_vmxDict["ethernet" + str(netCount) + ".networkname"]}) + vmxDict.update({"ethernet" + str(netCount) + ".addresstype": "generated"}) + vmxDict.update({"ethernet" + str(netCount) + ".present": "TRUE"}) + netCount = netCount + 1 + + ### Disk cloning - clone all disks from source + response, cookies = self.soap_client.send_req('<_this type="PropertyCollector">ha-property-collectorVirtualMachinefalselayout' + str(template_moid) + 'false') + xmltodictresponse = xmltodict.parse(response.read(), force_list='disk') + srcDiskFiles = [disk.get('diskFile') for disk in xmltodictresponse['soapenv:Envelope']['soapenv:Body']['RetrievePropertiesExResponse']['returnval']['objects']['propSet']['val']['disk']] + for srcDiskFile in srcDiskFiles: + srcDiskFileInfo = re.search('^\[(?P.*?)\] *(?P.*\/(?P(?P.*?)(?:--(?P.*?))?\.vmdk))$', srcDiskFile) + diskTypeKey = next((key for key, val in template_vmxDict.items() if val == srcDiskFileInfo.group('filepath')), None) + + if re.search('scsi', diskTypeKey): + controllerTypeStr = "scsi0:" + else: + controllerTypeStr = "sata0:" + + # See if vmTemplate disk exists + try: + (stdin, stdout, stderr) = self.esxiCnx.exec_command("stat /vmfs/volumes/" + srcDiskFileInfo.group('datastore') + "/" + srcDiskFileInfo.group('fulldiskpath')) + except IOError as e: + return (srcDiskFileInfo.group('fulldiskpath') + " not found!") + else: + if diskCount == 0: + disk_filename = self.name + "--boot.vmdk" + else: + if 'diskname_suffix' in srcDiskFileInfo.groupdict() and srcDiskFileInfo.group('diskname_suffix'): + disk_filename = self.name + "--" + srcDiskFileInfo.group('diskname_suffix') + ".vmdk" + else: + disk_filename = self.name + ".vmdk" + self.esxiCnx.exec_command("vmkfstools -i /vmfs/volumes/" + srcDiskFileInfo.group('datastore') + "/" + srcDiskFileInfo.group('fulldiskpath') + " -d thin " + vmPathDest + "/" + disk_filename) + + vmxDict.update({controllerTypeStr + str(diskCount) + ".devicetype": "scsi-hardDisk"}) + vmxDict.update({controllerTypeStr + str(diskCount) + ".present": "TRUE"}) + vmxDict.update({controllerTypeStr + str(diskCount) + ".filename": disk_filename}) + diskCount = diskCount + 1 + + else: + return (vmTemplate + " not found!") + + ## Now add remaining settings, overriding template copies. + + # Generic settings + if guest_id: + vmxDict.update({"guestos": guest_id}) + vmxDict.update({"displayname": self.name}) + vmxDict.update({"vm.createdate": time.time()}) + + if annotation: + vmxDict.update({"annotation": annotation}) + + # Hardware settings + if 'version' in hardware: + vmxDict.update({"virtualhw.version": hardware['version']}) + if 'memory_mb' in hardware: + vmxDict.update({"memsize": hardware['memory_mb']}) + if 'num_cpus' in hardware: + vmxDict.update({"numvcpus": hardware['num_cpus']}) + if 'num_cpu_cores_per_socket' in hardware: + vmxDict.update({"cpuid.coresPerSocket": hardware['num_cpu_cores_per_socket']}) + if 'hotadd_cpu' in hardware: + vmxDict.update({"vcpu.hotadd": hardware['hotadd_cpu']}) + if 'hotadd_memory' in hardware: + vmxDict.update({"mem.hotadd": hardware['hotadd_memory']}) + if 'memory_reservation_lock' in hardware: + vmxDict.update({"sched.mem.pin": hardware['memory_reservation_lock']}) + + # CDROM settings + if cdrom['type'] == 'client': + (stdin, stdout, stderr) = self.esxiCnx.exec_command("find /vmfs/devices/cdrom/ -mindepth 1 ! -type l") + cdrom_dev = stdout.read().decode('UTF-8').lstrip("\r\n").rstrip(" \r\n") + vmxDict.update({"ide0:0.devicetype": "atapi-cdrom"}) + vmxDict.update({"ide0:0.filename": cdrom_dev}) + vmxDict.update({"ide0:0.present": "TRUE"}) + elif cdrom['type'] == 'iso': + if 'iso_path' in cdrom: + vmxDict.update({"ide0:0.devicetype": "cdrom-image"}) + vmxDict.update({"ide0:0.filename": cdrom['iso_path']}) + vmxDict.update({"ide0:0.present": "TRUE"}) + vmxDict.update({"ide0:0.startconnected": "TRUE"}) + + # Network settings + cloudinit_nets = {"version": 2} + for netCount in range(0, len(networks)): + vmxDict.update({"ethernet" + str(netCount) + ".virtualdev": networks[netCount]['virtualDev']}) + vmxDict.update({"ethernet" + str(netCount) + ".networkname": networks[netCount]['networkName']}) + if "macAddress" in networks[netCount]: + vmxDict.update({"ethernet" + str(netCount) + ".addresstype": "static"}) + vmxDict.update({"ethernet" + str(netCount) + ".address": networks[netCount]['macAddress']}) + vmxDict.update({"ethernet" + str(netCount) + ".checkmacaddress": "FALSE"}) + else: + vmxDict.update({"ethernet" + str(netCount) + ".addresstype": "generated"}) + vmxDict.update({"ethernet" + str(netCount) + ".present": "TRUE"}) + if "cloudinit_netplan" in networks[netCount]: + cloudinit_nets.update(networks[netCount]['cloudinit_netplan']) + + # Add cloud-init metadata (hostname & network) + cloudinit_metadata = {"local-hostname": self.name} + if cloudinit_nets['ethernets'].keys(): + # Force guest to use the MAC address as the DHCP identifier, in case the machine-id is not reset for each clone + for cloudeth in cloudinit_nets['ethernets'].keys(): + cloudinit_nets['ethernets'][cloudeth].update({"dhcp-identifier": "mac"}) + # Add the metadata + cloudinit_metadata.update({"network": base64.b64encode(yaml.dump(cloudinit_nets, width=4096, encoding='utf-8')).decode('ascii'), "network.encoding": "base64"}) + vmxDict.update({"guestinfo.metadata": base64.b64encode(yaml.dump(cloudinit_metadata, width=4096, encoding='utf-8')).decode('ascii'), "guestinfo.metadata.encoding": "base64"}) + + # Add cloud-init userdata (must be in MIME multipart format) + if cloudinit_userdata and len(cloudinit_userdata): + import sys + from email.mime.multipart import MIMEMultipart + from email.mime.text import MIMEText + combined_message = MIMEMultipart() + sub_message = MIMEText(yaml.dump({"users": cloudinit_userdata}, width=4096, encoding='utf-8'), "cloud-config", sys.getdefaultencoding()) + sub_message.add_header('Content-Disposition', 'attachment; filename="cloud-config.yaml"') + combined_message.attach(sub_message) + if sys.version_info >= (3, 0): + vmxDict.update({"guestinfo.userdata": base64.b64encode(combined_message.as_bytes()).decode('ascii'), "guestinfo.userdata.encoding": "base64"}) + else: + vmxDict.update({"guestinfo.userdata": base64.b64encode(combined_message.as_string()).decode('ascii'), "guestinfo.userdata.encoding": "base64"}) + + ### Disk create + # If the first disk doesn't exist, create it + bootDisks = [bootDisk for bootDisk in disks if 'boot' in bootDisk] + if len(bootDisks) > 1: + return ("Muiltiple boot disks not allowed") + + if "scsi0:0.filename" not in vmxDict: + if len(bootDisks) == 1: + disk_filename = self.name + "--boot.vmdk" + (stdin, stdout, stderr) = self.esxiCnx.exec_command("vmkfstools -c " + str(bootDisks[0]['size_gb']) + "G -d " + bootDisks[0]['type'] + " " + vmPathDest + "/" + disk_filename) + + vmxDict.update({"scsi0:0.devicetype": "scsi-hardDisk"}) + vmxDict.update({"scsi0:0.present": "TRUE"}) + vmxDict.update({"scsi0:0.filename": disk_filename}) + diskCount = diskCount + 1 + if len(bootDisks) == 0: + return ("Boot disk parameters not defined for new VM") + else: + if len(bootDisks) == 1: + return ("Boot disk parameters defined for cloned VM. Ambiguous requirement - not supported.") + + # write the vmx + self.put_vmx(vmxDict, vmPathDest + "/" + self.name + ".vmx") + + # Register the VM + (stdin, stdout, stderr) = self.esxiCnx.exec_command("vim-cmd solo/registervm " + vmPathDest + "/" + self.name + ".vmx") + self.moid = int(stdout.readlines()[0]) + + # The logic used to update the disks is the same for an existing as a new VM. + self.update_vm(annotation=None, disks=disks) + + def update_vm(self, annotation, disks): + vmxPath, vmxDict = self.get_vmx(self.moid) + if annotation: + # Update the config (annotation) in the running VM + response, cookies = self.soap_client.send_req('<_this type="VirtualMachine">' + str(self.moid) + '' + annotation + '') + waitresp = self.soap_client.wait_for_task(xmltodict.parse(response.read())['soapenv:Envelope']['soapenv:Body']['ReconfigVM_TaskResponse']['returnval']['#text']) + if waitresp != 'success': + return ("Failed to ReconfigVM_Task: %s" % waitresp) + + # Now update the vmxFile on disk (should not be necessary, but for some reason, sometimes the ReconfigVM_Task does not flush config to disk). + vmxDict.update({"annotation": annotation}) + + if disks: + curDisks = [{"filename": vmxDict[scsiDisk], "volname": re.sub(r".*--([\w\d]+)\.vmdk", r"\1", vmxDict[scsiDisk])} for scsiDisk in sorted(vmxDict) if re.match(r"scsi0:\d\.filename", scsiDisk)] + curDisksCount = len(curDisks) + newDisks = [newDisk for newDisk in disks if ('boot' not in newDisk or newDisk['boot'] == False)] + for newDiskCount, newDisk in enumerate(newDisks): + scsiDiskIdx = newDiskCount + curDisksCount + disk_filename = self.name + "--" + newDisk['volname'] + ".vmdk" + + # Don't clone already-existing disks + try: + (stdin, stdout, stderr) = self.esxiCnx.exec_command("stat " + os.path.dirname(vmxPath) + "/" + disk_filename) + except IOError as e: + if 'src' in newDisk and newDisk['src'] is not None: + cloneSrcBackingFile = re.search('^\[(?P.*?)\] *(?P.*\/(?P(?P.*?)(?:--(?P.*?))?\.vmdk))$', newDisk['src']['backing_filename']) + try: + (stdin, stdout, stderr) = self.esxiCnx.exec_command("stat /vmfs/volumes/" + cloneSrcBackingFile.group('datastore') + "/" + cloneSrcBackingFile.group('fulldiskpath')) + except IOError as e: + return (cloneSrcBackingFile.group('fulldiskpath') + " not found!\n" + str(e)) + else: + if newDisk['src']['copy_or_move'] == 'copy': + self.esxiCnx.exec_command("vmkfstools -i /vmfs/volumes/" + cloneSrcBackingFile.group('datastore') + "/" + cloneSrcBackingFile.group('fulldiskpath') + " -d thin " + os.path.dirname(vmxPath) + "/" + disk_filename) + else: + self.esxiCnx.exec_command("vmkfstools -E /vmfs/volumes/" + cloneSrcBackingFile.group('datastore') + "/" + cloneSrcBackingFile.group('fulldiskpath') + " " + os.path.dirname(vmxPath) + "/" + disk_filename) + + else: + (stdin, stdout, stderr) = self.esxiCnx.exec_command("vmkfstools -c " + str(newDisk['size_gb']) + "G -d " + newDisk['type'] + " " + os.path.dirname(vmxPath) + "/" + disk_filename) + + # if this is a new disk, not a restatement of an existing disk: + if len(curDisks) >= newDiskCount + 2 and curDisks[newDiskCount + 1]['volname'] == newDisk['volname']: + pass + else: + vmxDict.update({"scsi0:" + str(scsiDiskIdx) + ".devicetype": "scsi-hardDisk"}) + vmxDict.update({"scsi0:" + str(scsiDiskIdx) + ".present": "TRUE"}) + vmxDict.update({"scsi0:" + str(scsiDiskIdx) + ".filename": disk_filename}) + curDisksCount = curDisksCount + 1 + + self.put_vmx(vmxDict, vmxPath) + self.esxiCnx.exec_command("vim-cmd vmsvc/reload " + str(self.moid)) + + # def update_vm_pyvmomi(self, annotation=None): + # if annotation: + # from pyVmomi import vim + # from pyVim.task import WaitForTask + # from pyVim import connect + # + # SI = connect.SmartConnectNoSSL(host=hostname, user=username, pwd=password, port=443) + # vm = SI.content.searchIndex.FindByDnsName(None, self.name, True) + # + # spec = vim.vm.ConfigSpec() + # spec.annotation = annotation + # task = vm.ReconfigVM_Task(spec) + # WaitForTask(task) + + # Delete the cloud-init guestinfo.metadata info from the .vmx file, otherwise it will be impossible to change the network configuration or hostname. + def delete_cloudinit(self): + vmxPath, vmxDict = self.get_vmx(self.moid) + if 'guestinfo.metadata' in vmxDict: + del vmxDict['guestinfo.metadata'] + if 'guestinfo.metadata.encoding' in vmxDict: + del vmxDict['guestinfo.metadata.encoding'] + if 'guestinfo.userdata' in vmxDict: + del vmxDict['guestinfo.userdata'] + if 'guestinfo.userdata.encoding' in vmxDict: + del vmxDict['guestinfo.userdata.encoding'] + + # write the vmx + self.put_vmx(vmxDict, vmxPath) + + +def main(): + argument_spec = { + "hostname": {"type": "str", "required": True}, + "username": {"type": "str", "required": True}, + "password": {"type": "str"}, + "name": {"type": "str"}, + "moid": {"type": "str"}, + "template": {"type": "str"}, + "state": {"type": "str", "default": 'present', "choices": ['absent', 'present', 'unchanged', 'rebootguest', 'poweredon', 'poweredoff', 'shutdownguest']}, + "force": {"type": "bool", "default": False}, + "datastore": {"type": "str"}, + "annotation": {"type": "str", "default": ""}, + "guest_id": {"type": "str", "default": "ubuntu-64"}, + "hardware": {"type": "dict", "default": {"version": "15", "num_cpus": "2", "memory_mb": "2048", "num_cpu_cores_per_socket": "1", "hotadd_cpu": "False", "hotadd_memory": "False", "memory_reservation_lock": "False"}}, + "cloudinit_userdata": {"type": "list", "default": []}, + "disks": {"type": "list", "default": [{"boot": True, "size_gb": 16, "type": "thin"}]}, + "cdrom": {"type": "dict", "default": {"type": "client"}}, + "networks": {"type": "list", "default": [{"networkName": "VM Network", "virtualDev": "vmxnet3"}]}, + "customvalues": {"type": "list", "default": []}, + "wait": {"type": "bool", "default": True}, + "wait_timeout": {"type": "int", "default": 180} + } + + if not (len(sys.argv) > 1 and sys.argv[1] == "console"): + module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=True, required_one_of=[['name', 'moid']]) + else: + # For testing without Ansible (e.g on Windows) + module = cDummyAnsibleModule() + ## Update VM + module.params = { + "hostname": "192.168.1.3", + "username": "svc", + "password": sys.argv[2], + # "annotation": "{'Name': 'dougal-test-dev-sysdisks2-a0-1617548508', 'hosttype': 'sysdisks2', 'env': 'dev', 'cluster_name': 'dougal-test-dev', 'owner': 'dougal', 'cluster_suffix': '1617548508', 'lifecycle_state': 'retiring', 'maintenance_mode': 'false'}", + "annotation": None, + "disks": None, + "name": "cvtest-16-dd9032f65aef7-dev-sys-b0-1617726990", + "moid": None, + "state": "unchanged", + "wait_timeout": 180 + } + + # ## Delete VM + # module.params = { + # "hostname": "192.168.1.3", + # "username": "svc", + # "password": sys.argv[2], + # "name": "test-asdf", + # "moid": None, + # "state": "absent" + # } + # + # ## Clone VM + # module.params = { + # "hostname": "192.168.1.3", + # "username": "svc", + # "password": sys.argv[2], + # "annotation": None, + # # "annotation": "{'lifecycle_state': 'current', 'Name': 'test-prod-sys-a0-1589979249', 'cluster_suffix': '1589979249', 'hosttype': 'sys', 'cluster_name': 'test-prod', 'env': 'prod', 'owner': 'dougal'}", + # "cdrom": {"type": "client"}, + # "cloudinit_userdata": [], + # "customvalues": [], + # "datastore": "4tb-evo860-ssd", + # "disks": [], + # # "disks": [{"size_gb": 1, "type": "thin", "volname": "test"}], + # # "disks": [{"size_gb": 1, "type": "thin", "volname": "test", "src": {"backing_filename": "[4tb-evo860-ssd] testdisks-dev-sys-a0-1601204786/testdisks-dev-sys-a0-1601204786--test.vmdk", "copy_or_move": "move"}}], + # "force": False, + # "guest_id": "ubuntu-64", + # "hardware": {"memory_mb": "2048", "num_cpus": "2", "version": "15"}, + # "moid": None, + # "name": "dougal-test-dev-sys-a0-new", + # "networks": [{"cloudinit_netplan": {"ethernets": {"eth0": {"dhcp4": True}}}, "networkName": "VM Network", "virtualDev": "vmxnet3"}], + # "state": "present", + # "template": "dougal-test-dev-sys-a0-1617553110", + # "wait": True, + # "wait_timeout": 180 + # } + # + # ## Create blank VM + # module.params = { + # "hostname": "192.168.1.3", + # "username": "svc", + # "password": sys.argv[2], + # "name": "test-asdf", + # "annotation": "{'Name': 'test-asdf'}", + # "datastore": "4tb-evo860-ssd", + # "force": False, + # "moid": None, + # "template": None, + # "state": "present", + # "guest_id": "ubuntu-64", + # "hardware": {"version": "15", "num_cpus": "2", "memory_mb": "2048"}, + # "cloudinit_userdata": [], + # "disks": [{"boot": True, "size_gb": 16, "type": "thin"}, {"size_gb": 5, "type": "thin"}, {"size_gb": 2, "type": "thin"}], + # "cdrom": {"type": "iso", "iso_path": "/vmfs/volumes/4tb-evo860-ssd/ISOs/ubuntu-18.04.2-server-amd64.iso"}, + # "networks": [{"networkName": "VM Network", "virtualDev": "vmxnet3"}], + # "customvalues": [], + # "wait": True, + # "wait_timeout": 180, + # } + + iScraper = esxiFreeScraper(hostname=module.params['hostname'], + username=module.params['username'], + password=module.params['password'], + name=module.params['name'], + moid=module.params['moid']) + + if iScraper.moid is None and iScraper.name is None: + module.fail_json(msg="If VM doesn't already exist, you must provide a name for it") + + # Check if the VM exists before continuing + if module.params['state'] == 'unchanged': + if iScraper.moid is not None: + updateVmResult = iScraper.update_vm(annotation=module.params['annotation'], disks=module.params['disks']) + if updateVmResult != None: + module.fail_json(msg=updateVmResult) + module.exit_json(changed=True, meta={"msg": "Shutdown " + iScraper.name + ": " + str(iScraper.moid)}) + else: + module.fail_json(msg="VM doesn't exist.") + + elif module.params['state'] == 'shutdownguest': + if iScraper.moid: + iScraper.soap_client.send_req('<_this type="VirtualMachine">' + str(iScraper.moid) + '') + time_s = 60 + while time_s > 0: + (stdin, stdout, stderr) = iScraper.esxiCnx.exec_command("vim-cmd vmsvc/power.getstate " + str(iScraper.moid)) + if re.search('Powered off', stdout.read().decode('UTF-8')) is not None: + break + else: + time.sleep(1) + time_s = time_s - 1 + module.exit_json(changed=True, meta={"msg": "Shutdown " + iScraper.name + ": " + str(iScraper.moid)}) + else: + module.fail_json(msg="VM doesn't exist.") + + elif module.params['state'] == 'poweredon': + if iScraper.moid: + (stdin, stdout, stderr) = iScraper.esxiCnx.exec_command("vim-cmd vmsvc/power.getstate " + str(iScraper.moid)) + if re.search('Powered off', stdout.read().decode('UTF-8')) is not None: + response, cookies = iScraper.soap_client.send_req('<_this type="VirtualMachine">' + str(iScraper.moid) + '') + if iScraper.soap_client.wait_for_task(xmltodict.parse(response.read())['soapenv:Envelope']['soapenv:Body']['PowerOnVM_TaskResponse']['returnval']['#text'], int(module.params['wait_timeout'])) != 'success': + module.fail_json(msg="Failed to PowerOnVM_Task") + module.exit_json(changed=True, meta={"msg": "Powered-on " + iScraper.name + ": " + str(iScraper.moid)}) + else: + module.exit_json(changed=False, meta={"msg": "VM " + iScraper.name + ": already on."}) + else: + module.fail_json(msg="VM doesn't exist.") + + elif module.params['state'] == 'poweredoff': + if iScraper.moid: + (stdin, stdout, stderr) = iScraper.esxiCnx.exec_command("vim-cmd vmsvc/power.getstate " + str(iScraper.moid)) + if re.search('Powered on', stdout.read().decode('UTF-8')) is not None: + response, cookies = iScraper.soap_client.send_req('<_this type="VirtualMachine">' + str(iScraper.moid) + '') + if iScraper.soap_client.wait_for_task(xmltodict.parse(response.read())['soapenv:Envelope']['soapenv:Body']['PowerOffVM_TaskResponse']['returnval']['#text'], int(module.params['wait_timeout'])) != 'success': + module.fail_json(msg="Failed to PowerOffVM_Task") + module.exit_json(changed=True, meta={"msg": "Powered-off " + iScraper.name + ": " + str(iScraper.moid)}) + else: + module.exit_json(changed=False, meta={"msg": "VM " + iScraper.name + ": already off."}) + else: + module.fail_json(msg="VM doesn't exist.") + + elif module.params['state'] == 'absent': + if iScraper.moid: + # Turn off (ignoring failures), then destroy + response, cookies = iScraper.soap_client.send_req('<_this type="VirtualMachine">' + str(iScraper.moid) + '') + iScraper.soap_client.wait_for_task(xmltodict.parse(response.read())['soapenv:Envelope']['soapenv:Body']['PowerOffVM_TaskResponse']['returnval']['#text'], int(module.params['wait_timeout'])) + + response, cookies = iScraper.soap_client.send_req('<_this type="VirtualMachine">' + str(iScraper.moid) + '') + if iScraper.soap_client.wait_for_task(xmltodict.parse(response.read())['soapenv:Envelope']['soapenv:Body']['Destroy_TaskResponse']['returnval']['#text'], int(module.params['wait_timeout'])) != 'success': + module.fail_json(msg="Failed to Destroy_Task") + module.exit_json(changed=True, meta={"msg": "Deleted " + iScraper.name + ": " + str(iScraper.moid)}) + else: + module.exit_json(changed=False, meta={"msg": "VM " + iScraper.name + ": already absent."}) + + elif module.params['state'] == 'rebootguest': + if iScraper.moid: + (stdin, stdout, stderr) = iScraper.esxiCnx.exec_command("vim-cmd vmsvc/power.getstate " + str(iScraper.moid)) + if re.search('Powered off', stdout.read().decode('UTF-8')) is not None: + response, cookies = iScraper.soap_client.send_req('<_this type="VirtualMachine">' + str(iScraper.moid) + '') + if iScraper.soap_client.wait_for_task(xmltodict.parse(response.read())['soapenv:Envelope']['soapenv:Body']['PowerOffVM_TaskResponse']['returnval']['#text'], int(module.params['wait_timeout'])) != 'success': + module.fail_json(msg="Failed to PowerOnVM_Task") + else: + response, cookies = iScraper.soap_client.send_req('<_this type="VirtualMachine">' + str(iScraper.moid) + '') + module.exit_json(changed=True, meta={"msg": "Rebooted " + iScraper.name + ": " + str(iScraper.moid)}) + else: + module.fail_json(msg="VM doesn't exist.") + + elif module.params['state'] == 'present': + exit_args = {} + # If the VM already exists, and the 'force' flag is set, then we delete it (and recreate it) + if iScraper.moid and module.params['force']: + # Turn off (ignoring failures), then destroy + response, cookies = iScraper.soap_client.send_req('<_this type="VirtualMachine">' + str(iScraper.moid) + '') + if iScraper.soap_client.wait_for_task(xmltodict.parse(response.read())['soapenv:Envelope']['soapenv:Body']['PowerOffVM_TaskResponse']['returnval']['#text'], int(module.params['wait_timeout'])) != 'success': + module.fail_json(msg="Failed to PowerOffVM_Task (prior to Destroy_Task") + + response, cookies = iScraper.soap_client.send_req('<_this type="VirtualMachine">' + str(iScraper.moid) + '') + if iScraper.soap_client.wait_for_task(xmltodict.parse(response.read())['soapenv:Envelope']['soapenv:Body']['Destroy_TaskResponse']['returnval']['#text'], int(module.params['wait_timeout'])) != 'success': + module.fail_json(msg="Failed to Destroy_Task") + iScraper.moid = None + + # If the VM doesn't exist, create it. + if iScraper.moid is None: + # If we're cloning, ensure template VM is powered off. + if module.params['template'] is not None: + iScraperTemplate = esxiFreeScraper(hostname=module.params['hostname'], username=module.params['username'], password=module.params['password'], name=module.params['template'], moid=None) + (stdin, stdout, stderr) = iScraper.esxiCnx.exec_command("vim-cmd vmsvc/power.getstate " + str(iScraperTemplate.moid)) + if re.search('Powered off', stdout.read().decode('UTF-8')) is not None: + createVmResult = iScraper.create_vm(module.params['template'], module.params['annotation'], module.params['datastore'], module.params['hardware'], module.params['guest_id'], module.params['disks'], module.params['cdrom'], module.params['customvalues'], module.params['networks'], module.params['cloudinit_userdata']) + if createVmResult != None: + module.fail_json(msg="Failed to create_vm: %s" % createVmResult) + else: + module.fail_json(msg="Template VM must be powered off before cloning") + + else: + updateVmResult = iScraper.update_vm(annotation=module.params['annotation'], disks=module.params['disks']) + if updateVmResult != None: + module.fail_json(msg=updateVmResult) + + (stdin, stdout, stderr) = iScraper.esxiCnx.exec_command("vim-cmd vmsvc/power.getstate " + str(iScraper.moid)) + if re.search('Powered off', stdout.read().decode('UTF-8')) is not None: + response, cookies = iScraper.soap_client.send_req('<_this type="VirtualMachine">' + str(iScraper.moid) + '') + if iScraper.soap_client.wait_for_task(xmltodict.parse(response.read())['soapenv:Envelope']['soapenv:Body']['PowerOnVM_TaskResponse']['returnval']['#text'], int(module.params['wait_timeout'])) != 'success': + module.fail_json(msg="Failed to PowerOnVM_Task") + + isChanged = True + + ## Delete the cloud-init config + iScraper.delete_cloudinit() + + ## Wait for IP address and hostname to be advertised by the VM (via open-vm-tools) + if "wait" in module.params and module.params['wait']: + time_s = int(module.params['wait_timeout']) + while time_s > 0: + (stdin, stdout, stderr) = iScraper.esxiCnx.exec_command("vim-cmd vmsvc/get.guest " + str(iScraper.moid)) + guest_info = stdout.read().decode('UTF-8') + vm_params = re.search('\s*hostName\s*=\s*\"?(?P.*?)\"?,.*\n\s*ipAddress\s*=\s*\"?(?P.*?)\"?,.*', guest_info) + if vm_params and vm_params.group('vm_ip') != "" and vm_params.group('vm_hostname') != "": + break + else: + time.sleep(1) + time_s = time_s - 1 + + module.exit_json(changed=isChanged, + guest_info=guest_info, + hostname=vm_params.group('vm_hostname'), + ip_address=vm_params.group('vm_ip'), + name=module.params['name'], + moid=iScraper.moid) + else: + module.exit_json(changed=isChanged, + hostname="", + ip_address="", + name=module.params['name'], + moid=iScraper.moid) + + else: + module.exit_json(changed=False, meta={"msg": "No state."}) + + +if __name__ == '__main__': + main() diff --git a/_dependencies/library/esxifree_guest_LICENSE b/_dependencies/library/esxifree_guest_LICENSE new file mode 100644 index 00000000..7dce5362 --- /dev/null +++ b/_dependencies/library/esxifree_guest_LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2021, Dougal Seeley +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/_dependencies/library/esxifree_guest_README.md b/_dependencies/library/esxifree_guest_README.md new file mode 100644 index 00000000..9cdceaa8 --- /dev/null +++ b/_dependencies/library/esxifree_guest_README.md @@ -0,0 +1,53 @@ +# esxifree_guest +https://github.com/dseeley/esxifree_guest + +This module can be used to create new ESXi virtual machines, including cloning from templates or other virtual machines. + +It does so using direct SOAP calls and Paramiko SSH to the host - without using the vSphere API - meaning it can be used on the free hypervisor. + +## Configuration +Your ESXi host needs some config: ++ Enable SSH + + Inside the web UI, navigate to “Manage”, then the “Services” tab. Find the entry called: “TSM-SSH”, and enable it. ++ Enable “Guest IP Hack” + + `esxcli system settings advanced set -o /Net/GuestIPHack -i 1` ++ Open VNC Ports on the Firewall + ``` + Packer connects to the VM using VNC, so we’ll open a range of ports to allow it to connect to it. + + First, ensure we can edit the firewall configuration: + + chmod 644 /etc/vmware/firewall/service.xml + chmod +t /etc/vmware/firewall/service.xml + Then append the range we want to open to the end of the file: + + + packer-vnc + + inbound + tcp + dst + + 5900 + 6000 + + + true + true + + Finally, restore the permissions and reload the firewall: + + chmod 444 /etc/vmware/firewall/service.xml + esxcli network firewall refresh + ``` + +## Requirements ++ python 3 ++ paramiko ++ Any base-images from which clones are to be made must have cloud-init and [`cloud-init-vmware-guestinfo`](https://github.com/vmware/cloud-init-vmware-guestinfo) installed + +## Execution +This can be run as an Ansible module (see inline documentation), or from the console: +```bash +python3 ./esxifree_guest.py console +``` \ No newline at end of file diff --git a/_dependencies/library/esxifree_guest_info.py b/_dependencies/library/esxifree_guest_info.py new file mode 100644 index 00000000..e7da599b --- /dev/null +++ b/_dependencies/library/esxifree_guest_info.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright (c) 2021, Dougal Seeley +# https://github.com/dseeley/esxifree_guest +# BSD 3-Clause License + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ['preview'], 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: esxifree_guest_info +short_description: Retrieves virtual machine info in ESXi without a dependency on the vSphere/ vCenter API. +description: > + This module can be used to retrieve virtual machine info. When fetching all VM info, does so atomically (a single SOAP call), to prevent race conditions. +version_added: '2.9' +author: +- Dougal Seeley (ansible@dougalseeley.com) +requirements: +- python >= 2.7 +- xmltodict +notes: + - Please make sure that the user used for esxifree_guest should have correct level of privileges. + - Tested on vSphere 7.0.2 +options: + hostname: + description: + - The hostname or IP address of the ESXi server. + required: true + type: str + username: + description: + - The username to access the ESXi server at C(hostname). + required: true + type: str + password: + description: + - The password of C(username) for the ESXi server, or the password for the private key (if required). + required: true + type: str + name: + description: + - Name of the virtual machine to retrieve (optional). + - Virtual machine names in ESXi are unique + - This parameter is case sensitive. + type: str + moid: + description: + - Managed Object ID of the virtual machine (optional). + type: str +''' +EXAMPLES = r''' +- name: Get virtual machine for ALL VMs + esxifree_guest_info: + hostname: "192.168.1.3" + username: "svc" + password: "my_passsword" + delegate_to: localhost + +- name: Get virtual machine for specific VM + esxifree_guest_info: + hostname: "192.168.1.3" + username: "svc" + password: "my_passsword" + name: "my_vm" + delegate_to: localhost +''' + +RETURN = r''' +instance: + description: metadata about the virtual machine + returned: always + type: dict + sample: None +''' + +import json +import re +import sys +import time +import xmltodict + +# For the soap client +try: + from urllib.request import Request, build_opener, HTTPSHandler, HTTPCookieProcessor + from urllib.response import addinfourl + from urllib.error import HTTPError + from http.cookiejar import CookieJar + from http.client import HTTPResponse +except ImportError: + from urllib2 import Request, build_opener, HTTPError, HTTPSHandler, HTTPCookieProcessor, addinfourl + from cookielib import CookieJar + from httplib import HTTPResponse +import ssl + + +try: + from ansible.module_utils.basic import AnsibleModule +except: + # For testing without Ansible (e.g on Windows) + class cDummyAnsibleModule(): + def __init__(self): + self.params={} + def exit_json(self, changed, **kwargs): + print(changed, json.dumps(kwargs, sort_keys=True, indent=4, separators=(',', ': '))) + def fail_json(self, msg): + print("Failed: " + msg) + exit(1) + + +# Executes soap requests on the remote host. +class vmw_soap_client(object): + def __init__(self, host, username, password): + self.vmware_soap_session_cookie = None + self.host = host + response, cookies = self.send_req("<_this>ServiceInstance") + xmltodictresponse = xmltodict.parse(response.read()) + sessionManager_name = xmltodictresponse['soapenv:Envelope']['soapenv:Body']['RetrieveServiceContentResponse']['returnval']['sessionManager']['#text'] + + response, cookies = self.send_req("<_this>" + sessionManager_name + "" + username + "" + password + "") + self.vmware_soap_session_cookie = cookies['vmware_soap_session'].value + + def send_req(self, envelope_body=None): + envelope = '' + '' + str(envelope_body) + '' + cj = CookieJar() + req = Request( + url='https://' + self.host + '/sdk/vimService.wsdl', data=envelope.encode(), + headers={"Content-Type": "text/xml", "SOAPAction": "urn:vim25/6.7.3", "Accept": "*/*", "Cookie": "vmware_client=VMware; vmware_soap_session=" + str(self.vmware_soap_session_cookie)}) + + opener = build_opener(HTTPSHandler(context=ssl._create_unverified_context()), HTTPCookieProcessor(cj)) + num_send_attempts = 3 + for send_attempt in range(num_send_attempts): + try: + response = opener.open(req, timeout=30) + except HTTPError as err: + response = str(err) + except: + if send_attempt < num_send_attempts - 1: + time.sleep(1) + continue + else: + raise + break + + cookies = {i.name: i for i in list(cj)} + return (response[0] if isinstance(response, list) else response, cookies) # If the cookiejar contained anything, we get a list of two responses + + +class esxiFreeScraper(object): + def __init__(self, hostname, username='root', password=None): + self.soap_client = vmw_soap_client(host=hostname, username=username, password=password) + + def get_vm_info(self, name=None, moid=None): + if moid: + response, cookies = self.soap_client.send_req('<_this type="PropertyCollector">ha-property-collectorVirtualMachinetrue' + str(moid) + 'false') + xmltodictresponse = xmltodict.parse(response.read()) + return (self.parse_vm(xmltodictresponse['soapenv:Envelope']['soapenv:Body']['RetrievePropertiesExResponse']['returnval']['objects'])) + elif name: + virtual_machines = self.get_all_vm_info() + return ([vm for vm in virtual_machines if vm['hw_name'] == name][0]) + + def get_all_vm_info(self): + response, cookies = self.soap_client.send_req('<_this type="PropertyCollector">ha-property-collectorVirtualMachinefalsenameconfigconfigStatusdatastoreguestlayoutlayoutExruntimeha-folder-vmtraverseChildFolderchildEntity traverseChildDatacentervmFoldertraverseChild ') + xmltodictresponse = xmltodict.parse(response.read()) + + virtual_machines = [] + for vm_instance in xmltodictresponse['soapenv:Envelope']['soapenv:Body']['RetrievePropertiesExResponse']['returnval']['objects']: + virtual_machines.append(self.parse_vm(vm_instance)) + + return (virtual_machines) + + def _getObjSafe(self, inDict, *keys): + for key in keys: + try: inDict = inDict[key] + except KeyError: return None + return inDict + + def parse_vm(self, vmObj): + configObj = [propSetObj for propSetObj in vmObj['propSet'] if propSetObj['name'] == 'config'][0]['val'] + runtimeObj = [propSetObj for propSetObj in vmObj['propSet'] if propSetObj['name'] == 'runtime'][0]['val'] + guestObj = [propSetObj for propSetObj in vmObj['propSet'] if propSetObj['name'] == 'guest'][0]['val'] + layoutExObj = [propSetObj for propSetObj in vmObj['propSet'] if propSetObj['name'] == 'layoutEx'][0]['val'] + newObj = {} + + newObj.update({"advanced_settings": {advObj['key']: advObj['value'].get('#text') for advObj in configObj['extraConfig']}}) + newObj.update({"annotation": configObj['annotation']}) + newObj.update({"consolidationNeeded": runtimeObj['consolidationNeeded']}) + newObj.update({"guest_tools_status": guestObj['toolsRunningStatus'] if 'toolsRunningStatus' in guestObj else None}) + newObj.update({"guest_tools_version": guestObj['toolsVersion'] if 'toolsVersion' in guestObj else None}) + newObj.update({"hw_cores_per_socket": configObj['hardware']['numCoresPerSocket']}) + newObj.update({"hw_datastores": [configObj['datastoreUrl']['name']]}) + newObj.update({"hw_files": [file.get('name') for file in layoutExObj['file'] if file.get('type') in ['config', 'nvram', 'diskDescriptor', 'snapshotList', 'log']]}) + newObj.update({"hw_guest_full_name": guestObj['guestFullName'] if 'guestFullName' in guestObj else None}) + newObj.update({"hw_guest_id": guestObj['guestId'] if 'guestId' in guestObj else None}) + newObj.update({"hw_is_template": configObj['template']}) + newObj.update({"hw_memtotal_mb": int(configObj['hardware']['memoryMB'])}) + newObj.update({"hw_name": [propSetObj for propSetObj in vmObj['propSet'] if propSetObj['name'] == 'name'][0]['val'].get('#text')}) + newObj.update({"hw_power_status": runtimeObj['powerState']}) + newObj.update({"hw_processor_count": int(configObj['hardware']['numCPU'])}) + newObj.update({"hw_product_uuid": configObj['uuid']}) + newObj.update({"hw_version": configObj['version']}) + newObj.update({"ipv4": guestObj['ipAddress'] if 'ipAddress' in guestObj else None}) + newObj.update({"moid": vmObj['obj'].get('#text')}) + + guest_disk_info = [] + for virtualDiskObj in [diskObj for diskObj in configObj['hardware']['device'] if diskObj['@xsi:type'] == 'VirtualDisk']: + guest_disk_info.append({ + "backing_datastore": re.sub(r'^\[(.*?)\].*$', r'\1', virtualDiskObj['backing']['fileName']), + "backing_disk_mode": virtualDiskObj['backing']['diskMode'], + "backing_diskmode": virtualDiskObj['backing']['diskMode'], + "backing_eagerlyscrub": self._getObjSafe(virtualDiskObj, 'backing', 'eagerlyScrub'), + "backing_filename": virtualDiskObj['backing']['fileName'], + "backing_thinprovisioned": virtualDiskObj['backing']['thinProvisioned'], + "backing_type": re.sub(r'^VirtualDisk(.*?)BackingInfo$', r'\1', virtualDiskObj['backing']['@xsi:type']), + "backing_uuid": self._getObjSafe(virtualDiskObj, 'backing', 'uuid'), + "backing_writethrough": virtualDiskObj['backing']['writeThrough'], + "capacity_in_bytes": int(virtualDiskObj['capacityInBytes']), + "capacity_in_kb": int(virtualDiskObj['capacityInKB']), + "controller_key": virtualDiskObj['controllerKey'], + "controller_bus_number": [deviceObj['busNumber'] for deviceObj in configObj['hardware']['device'] if deviceObj['key'] == virtualDiskObj['controllerKey']][0], + "controller_type": [deviceObj['@xsi:type'] for deviceObj in configObj['hardware']['device'] if deviceObj['key'] == virtualDiskObj['controllerKey']][0], + "key": virtualDiskObj['key'], + "label": virtualDiskObj['deviceInfo']['label'], + "summary": virtualDiskObj['deviceInfo']['summary'], + "unit_number": int(virtualDiskObj['unitNumber']) + }) + newObj.update({"guest_disk_info": guest_disk_info}) + + return newObj + + +def main(): + argument_spec = { + "hostname": {"type": "str", "required": True}, + "username": {"type": "str", "required": True}, + "password": {"type": "str", "required": True}, + "name": {"type": "str"}, + "moid": {"type": "str"} + } + + if not (len(sys.argv) > 1 and sys.argv[1] == "console"): + module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=True) + else: + # For testing without Ansible (e.g on Windows) + module = cDummyAnsibleModule() + ## Update VM + module.params = { + "hostname": "192.168.1.3", + "username": "svc", + "password": sys.argv[2], + "name": None, # "parsnip-prod-sys-a0-1616868999", + "moid": None # 350 + } + + iScraper = esxiFreeScraper(hostname=module.params['hostname'], username=module.params['username'], password=module.params['password']) + + if ("moid" in module.params and module.params['name']) or ("name" in module.params and module.params['moid']): + vm_info = iScraper.get_vm_info(name=module.params['name'], moid=module.params['moid']) + else: + vm_info = iScraper.get_all_vm_info() + + module.exit_json(changed=False, virtual_machines=vm_info) + + +if __name__ == '__main__': + main() diff --git a/_dependencies/tasks/main.yml b/_dependencies/tasks/main.yml index 80db1ae5..3caa1de0 100644 --- a/_dependencies/tasks/main.yml +++ b/_dependencies/tasks/main.yml @@ -30,7 +30,7 @@ - name: Preflight check block: - - assert: { that: "ansible_version.full is version_compare('2.9.6', '>=') and ansible_version.full is version_compare('2.10.6', '<=')", fail_msg: "2.10.6 >= Ansible >= 2.9.6 required." } #2.10.7 has issue with AWS DNS: https://github.com/ansible-collections/community.aws/issues/523 + - assert: { that: "ansible_version.full is version_compare('2.10', '>=') and ansible_version.full is version_compare('2.10.6', '<=')", fail_msg: "2.10.6 >= Ansible >= 2.10 required." } #2.10.7 has issue with AWS DNS: https://github.com/ansible-collections/community.aws/issues/523 - assert: { that: "app_name is defined and app_name != ''", fail_msg: "Please define app_name" } - assert: { that: "app_class is defined and app_class != ''", fail_msg: "Please define app_class" } - assert: { that: "cluster_vars is defined", fail_msg: "Please define cluster_vars" } diff --git a/clean/tasks/azure.yml b/clean/tasks/azure.yml new file mode 100644 index 00000000..a8480f7c --- /dev/null +++ b/clean/tasks/azure.yml @@ -0,0 +1,146 @@ +--- + +- name: clean/azure | clean vms (and all dependent infrastructure) + block: + - name: clean/azure | Delete VMs (and all attached infra (NIC/IP/Storage)) asynchronously + azure.azcollection.azure_rm_virtualmachine: + client_id: "{{cluster_vars[buildenv].azure_client_id}}" + secret: "{{cluster_vars[buildenv].azure_secret}}" + subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" + tenant: "{{cluster_vars[buildenv].azure_tenant}}" + resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" + name: "{{item.name}}" + remove_on_absent: ["all"] + state: absent + register: r__azure_rm_virtualmachine + loop: "{{ hosts_to_clean }}" + async: 7200 + poll: 0 + + - name: clean/azure | Wait for instance deletion to complete + async_status: { jid: "{{ item.ansible_job_id }}" } + register: r__async_status__azure_rm_virtualmachine + until: r__async_status__azure_rm_virtualmachine.finished + delay: 3 + retries: 300 + with_items: "{{r__azure_rm_virtualmachine.results}}" + when: hosts_to_clean | length + + +#### ALTERNATE - IF NOT RELYING ON ANSIBLE-CREATED VMs +#- name: clean/azure | clean vms +# block: +# - name: clean/azure | Get instance resource info +# azure.azcollection.azure_rm_resource_info: +# client_id: "{{cluster_vars[buildenv].azure_client_id}}" +# secret: "{{cluster_vars[buildenv].azure_secret}}" +# subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" +# tenant: "{{cluster_vars[buildenv].azure_tenant}}" +# resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" +# resource_name: "{{ item.name }}" +# resource_type: VirtualMachines +# provider: Compute +# with_items: "{{ hosts_to_clean }}" +# register: r__azure_rm_resource_info__vm +# async: 7200 +# poll: 0 +# +# - name: clean/azure | Wait for instance resource info (to get Zone info) +# async_status: { jid: "{{ item.ansible_job_id }}" } +# register: r__async_status__azure_rm_resource_info__vm +# until: r__async_status__azure_rm_resource_info__vm.finished +# delay: 3 +# retries: 300 +# with_items: "{{r__azure_rm_resource_info__vm.results}}" +# +# +# - name: clean/azure | Delete VMs asynchronously +# azure.azcollection.azure_rm_virtualmachine: +# client_id: "{{cluster_vars[buildenv].azure_client_id}}" +# secret: "{{cluster_vars[buildenv].azure_secret}}" +# subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" +# tenant: "{{cluster_vars[buildenv].azure_tenant}}" +# resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" +# name: "{{item.name}}" +# remove_on_absent: ["all_autocreated"] +# state: absent +# register: r__azure_rm_virtualmachine +# loop: "{{ hosts_to_clean }}" +# async: 7200 +# poll: 0 +# +# - name: clean/azure | Wait for instance deletion to complete +# async_status: { jid: "{{ item.ansible_job_id }}" } +# register: r__async_status__azure_rm_virtualmachine +# until: r__async_status__azure_rm_virtualmachine.finished +# delay: 3 +# retries: 300 +# with_items: "{{r__azure_rm_virtualmachine.results}}" +# +# - name: create/azure | Delete managed disks +# azure.azcollection.azure_rm_manageddisk: +# client_id: "{{cluster_vars[buildenv].azure_client_id}}" +# secret: "{{cluster_vars[buildenv].azure_secret}}" +# subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" +# tenant: "{{cluster_vars[buildenv].azure_tenant}}" +# resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" +# name: "{{ item }}" +# state: absent +# loop: "{{ r__async_status__azure_rm_resource_info__vm.results | json_query(\"[[].response[].properties.storageProfile.dataDisks[].name, [].response[].properties.storageProfile.osDisk.name][]\") }}" +# register: r__aazure_rm_manageddisk +# async: 7200 +# poll: 0 +# +# - name: clean/azure | Wait for managed disk deletion +# async_status: { jid: "{{ item.ansible_job_id }}" } +# register: r__async_status__aazure_rm_manageddisk +# until: r__async_status__aazure_rm_manageddisk.finished +# delay: 3 +# retries: 300 +# with_items: "{{r__aazure_rm_manageddisk.results}}" +# +# +#- name: clean/azure | clean networking (when '-e clean=_all_') +# block: +# - name: clean/azure | Get network interface info (per instance) +# azure.azcollection.azure_rm_networkinterface_info: +# client_id: "{{cluster_vars[buildenv].azure_client_id}}" +# secret: "{{cluster_vars[buildenv].azure_secret}}" +# subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" +# tenant: "{{cluster_vars[buildenv].azure_tenant}}" +# resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" +# name: "{{ item | basename }}" +# loop: "{{ r__async_status__azure_rm_resource_info__vm.results | json_query(\"[].response[].properties.networkProfile.networkInterfaces[].id\") }}" +# register: r__azure_rm_networkinterface_info +# async: 7200 +# poll: 0 +# +# - name: clean/azure | Wait for network interface info +# async_status: { jid: "{{ item.ansible_job_id }}" } +# register: r__async_status__azure_rm_networkinterface_info +# until: r__async_status__azure_rm_networkinterface_info.finished +# delay: 3 +# retries: 300 +# with_items: "{{r__azure_rm_networkinterface_info.results}}" +# +# - name: clean/azure | Delete public ipaddresses +# azure.azcollection.azure_rm_publicipaddress: +# client_id: "{{cluster_vars[buildenv].azure_client_id}}" +# secret: "{{cluster_vars[buildenv].azure_secret}}" +# subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" +# tenant: "{{cluster_vars[buildenv].azure_tenant}}" +# resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" +# name: "{{ item.public_ip_id | basename }}" +# with_items: "{{ r__async_status__azure_rm_networkinterface_info.results | json_query(\"[].networkinterfaces[].ip_configurations[].public_ip_address\") }}" +# register: r__azure_rm_networkinterface +# async: 7200 +# poll: 0 +# +# - name: clean/azure | Wait for publicipaddress deletion +# async_status: { jid: "{{ item.ansible_job_id }}" } +# register: r__async_status__azure_rm_publicipaddress +# until: r__async_status__azure_rm_publicipaddress.finished +# delay: 3 +# retries: 300 +# with_items: "{{r__azure_rm_networkinterface.results}}" +# when: clean is defined and clean == '_all_' diff --git a/clean/tasks/dns.yml b/clean/tasks/dns.yml index 075e722a..333fe4d0 100644 --- a/clean/tasks/dns.yml +++ b/clean/tasks/dns.yml @@ -35,6 +35,21 @@ - name: clean/dns/route53 | Delete DNS entries block: +# - name: clean/dns/route53 | Get Zone +# route53_zone: +# aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}" +# aws_secret_key: "{{cluster_vars[buildenv].aws_secret_key}}" +# zone: "{{cluster_vars.dns_nameserver_zone}}" +# register: r__route53_zone +# +# - name: clean/dns/route53 | Get A records +# route53_info: +# query: record_sets +# hosted_zone_id: "{{ r__route53_zone.zone_id }}" +# start_record_name: "{{item.name}}.{{cluster_vars.dns_user_domain}}" +# register: r__route53_info +# with_items: "{{ hosts_to_clean }}" + - name: clean/dns/route53 | Get A records route53: aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}" @@ -47,6 +62,8 @@ register: r__route53_a with_items: "{{ hosts_to_clean }}" + - debug: msg={{r__route53_a}} + - name: clean/dns/route53 | Delete A records route53: aws_access_key: "{{cluster_vars[buildenv].aws_access_key}}" diff --git a/clean/tasks/esxifree.yml b/clean/tasks/esxifree.yml new file mode 100644 index 00000000..18ef67bd --- /dev/null +++ b/clean/tasks/esxifree.yml @@ -0,0 +1,25 @@ +--- + +- name: clean/esxifree + block: + - name: clean/esxifree | Delete VM + esxifree_guest: + hostname: "{{ cluster_vars.esxi_ip }}" + username: "{{ cluster_vars.username }}" + password: "{{ cluster_vars.password }}" + name: "{{item.name}}" + state: absent + with_items: "{{hosts_to_clean}}" + register: esxi_instances + run_once: true + async: 7200 + poll: 0 + + - name: clean/esxifree | Wait for VM deletion to complete + async_status: + jid: "{{ item.ansible_job_id }}" + register: esxi_jobs + until: esxi_jobs.finished + retries: 300 + with_items: "{{esxi_instances.results}}" + when: hosts_to_clean | length diff --git a/cluster_hosts/tasks/get_cluster_hosts_state_azure.yml b/cluster_hosts/tasks/get_cluster_hosts_state_azure.yml new file mode 100644 index 00000000..d8344c7d --- /dev/null +++ b/cluster_hosts/tasks/get_cluster_hosts_state_azure.yml @@ -0,0 +1,113 @@ +--- + +# Note: Azure, irritatingly, doesn't provide all the info we need for cluster_hosts_state/dynamic_inventory in one place. We have to run each of these, passing the results of the previous into the next. +# + VM info: azure_rm_virtualmachine_info +# + VM AZ info: azure_rm_resource_info +# + Private IP info: azure_rm_networkinterface_info +# + Public IP info: azure_rm_publicipaddress_info + +- name: get_cluster_hosts_state/azure | Get existing instance info + azure.azcollection.azure_rm_virtualmachine_info: + client_id: "{{cluster_vars[buildenv].azure_client_id}}" + secret: "{{cluster_vars[buildenv].azure_secret}}" + subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" + tenant: "{{cluster_vars[buildenv].azure_tenant}}" + resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" + tags: + - "cluster_name:{{cluster_name}}" + register: r__azure_rm_virtualmachine_info + delegate_to: localhost + run_once: true + +#- name: get_cluster_hosts_state/azure | r__azure_rm_virtualmachine_info +# debug: msg="{{r__azure_rm_virtualmachine_info}}" +# delegate_to: localhost +# run_once: true + +- name: get_cluster_hosts_state/azure | Get instance resource info (for VM AZ info) + azure.azcollection.azure_rm_resource_info: + client_id: "{{cluster_vars[buildenv].azure_client_id}}" + secret: "{{cluster_vars[buildenv].azure_secret}}" + subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" + tenant: "{{cluster_vars[buildenv].azure_tenant}}" + resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" + resource_name: "{{ item.name }}" + resource_type: VirtualMachines + provider: Compute + with_items: "{{ r__azure_rm_virtualmachine_info.vms }}" + register: r__azure_rm_resource_info + until: "(r__azure_rm_resource_info.response | json_query(\"[?properties.provisioningState!='Succeeded']|length(@)\")) == 0" + retries: 18 #3 mins + delay: 10 + delegate_to: localhost + run_once: true + +#- name: get_cluster_hosts_state/azure | r__azure_rm_resource_info +# debug: msg="{{r__azure_rm_resource_info}}" +# delegate_to: localhost +# run_once: true + +- name: get_cluster_hosts_state/azure | Get network interface info (per instance) + azure.azcollection.azure_rm_networkinterface_info: + client_id: "{{cluster_vars[buildenv].azure_client_id}}" + secret: "{{cluster_vars[buildenv].azure_secret}}" + subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" + tenant: "{{cluster_vars[buildenv].azure_tenant}}" + resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" + name: "{{ item.networkInterface | basename }}" + with_items: "{{ r__azure_rm_resource_info.results | to_json | from_json | json_query(\"[].{name: item.name, regionzone: join('-',[item.location,response[0].zones[0]]), tagslabels: item.tags, instance_id: item.id, instance_state: item.power_state, networkInterface: response[0].properties.networkProfile.networkInterfaces[0].id }\") }}" + register: r__azure_rm_networkinterface_info + delegate_to: localhost + run_once: true + async: 7200 + poll: 0 + +- name: get_cluster_hosts_state/azure | Wait for network interface info + async_status: { jid: "{{ item.ansible_job_id }}" } + register: r__async_status__azure_rm_networkinterface_info + until: r__async_status__azure_rm_networkinterface_info.finished + delay: 3 + retries: 300 + with_items: "{{r__azure_rm_networkinterface_info.results}}" + delegate_to: localhost + run_once: true + +#- name: get_cluster_hosts_state/azure | r__async_status__azure_rm_networkinterface_info +# debug: msg="{{r__async_status__azure_rm_networkinterface_info}}" +# delegate_to: localhost +# run_once: true + + +- name: get_cluster_hosts_state/azure | Get publicipaddress info (per instance) + azure.azcollection.azure_rm_publicipaddress_info: + client_id: "{{cluster_vars[buildenv].azure_client_id}}" + secret: "{{cluster_vars[buildenv].azure_secret}}" + subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" + tenant: "{{cluster_vars[buildenv].azure_tenant}}" + resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" + name: "{{ item.networkinterfaces[0].ip_configurations[0].public_ip_address | basename }}" + with_items: "{{ r__async_status__azure_rm_networkinterface_info.results }}" + register: r__azure_rm_publicipaddress_info + delegate_to: localhost + run_once: true + async: 7200 + poll: 0 + +- name: get_cluster_hosts_state/azure | Wait for publicipaddress info + async_status: { jid: "{{ item.ansible_job_id }}" } + register: r__async_status__azure_rm_publicipaddress_info + until: r__async_status__azure_rm_publicipaddress_info.finished + delay: 3 + retries: 300 + with_items: "{{r__azure_rm_publicipaddress_info.results}}" + delegate_to: localhost + run_once: true + +#- name: get_cluster_hosts_state/azure | r__async_status__azure_rm_publicipaddress_info +# debug: msg="{{r__async_status__azure_rm_publicipaddress_info}}" +# delegate_to: localhost +# run_once: true + +- name: get_cluster_hosts_state/azure | Set cluster_hosts_state + set_fact: + cluster_hosts_state: "{{r__async_status__azure_rm_publicipaddress_info.results | json_query(\"[].{name: item.item.item.item.name, regionzone: item.item.item.item.regionzone, tagslabels: item.item.item.item.tagslabels, instance_id: item.item.item.item.instance_id, instance_state: item.item.item.item.instance_state, ipv4: {private: item.item.networkinterfaces[0].ip_configurations[0].private_ip_address, public: publicipaddresses[0].ip_address} }\") }}" diff --git a/cluster_hosts/tasks/get_cluster_hosts_state_esxifree.yml b/cluster_hosts/tasks/get_cluster_hosts_state_esxifree.yml new file mode 100644 index 00000000..775be0a7 --- /dev/null +++ b/cluster_hosts/tasks/get_cluster_hosts_state_esxifree.yml @@ -0,0 +1,29 @@ +--- + +- name: get_cluster_hosts_state/esxifree | Get basic instance info of all vms + esxifree_guest_info: + username: "{{ cluster_vars.username }}" + password: "{{ cluster_vars.password }}" + hostname: "{{ cluster_vars.esxi_ip }}" + register: r__esxifree_guest_info + delegate_to: localhost + run_once: true + +## esxifree hosts must use the esxi 'annotations' field as json. They are stored as unconventional text in the vmx file, so must +## be converted into inline-json within the facts. If the annotation field is not convertible to json, then we don't consider this VM part of the cluster. +- name: get_cluster_hosts_state/esxifree | update r__esxifree_guest_info result with json-parsed annotations + set_fact: + r__esxifree_guest_info: | + {% set res = {'virtual_machines': []} -%} + {%- for result in r__esxifree_guest_info.virtual_machines -%} + {%- set loadloose_res = result.annotation | json_loads_loose -%} + {%- if loadloose_res | type_debug == 'dict' or loadloose_res | type_debug == 'list' -%} + {%- set _ = result.update({'annotation': loadloose_res}) -%} + {%- set _ = res.virtual_machines.append(result) -%} + {%- endif -%} + {%- endfor -%} + {{ res }} + +- name: get_cluster_hosts_state/esxifree | Set cluster_hosts_state + set_fact: + cluster_hosts_state: "{{ r__esxifree_guest_info.virtual_machines | json_query(\"[?annotation.cluster_name==`\" + cluster_name + \"`].{name: hw_name, regionzone: None, tagslabels: annotation, instance_id: moid, instance_state: hw_power_status, ipv4: {private: ipv4, public: null}, disk_info_cloud: guest_disk_info }\") }}" diff --git a/cluster_hosts/tasks/get_cluster_hosts_target_esxifree.yml b/cluster_hosts/tasks/get_cluster_hosts_target_esxifree.yml new file mode 100644 index 00000000..871c9a44 --- /dev/null +++ b/cluster_hosts/tasks/get_cluster_hosts_target_esxifree.yml @@ -0,0 +1,11 @@ +--- + +- name: get_cluster_hosts_target/esxifree | Update cluster_hosts_target with volname (derived from the mountpoint) + set_fact: + cluster_hosts_target: | + {%- for host in cluster_hosts_target -%} + {%- for hostvol in host.auto_volumes -%} + {%- set _dummy = hostvol.update({'volname': hostvol.mountpoint | regex_replace('.*\/(.*)', '\\1')}) -%} + {%- endfor %} + {%- endfor %} + {{ cluster_hosts_target }} diff --git a/config/tasks/disks_auto_aws_gcp.yml b/config/tasks/disks_auto_aws_gcp_azure.yml similarity index 63% rename from config/tasks/disks_auto_aws_gcp.yml rename to config/tasks/disks_auto_aws_gcp_azure.yml index a1452cf6..ccdfe2b4 100644 --- a/config/tasks/disks_auto_aws_gcp.yml +++ b/config/tasks/disks_auto_aws_gcp_azure.yml @@ -1,23 +1,23 @@ --- -- name: disks_auto_aws_gcp | cluster_hosts_target(inventory_hostname) +- name: disks_auto_aws_gcp_azure | cluster_hosts_target(inventory_hostname) debug: msg={{ cluster_hosts_target | json_query(\"[?hostname == '\" + inventory_hostname + \"'] \") }} -- name: disks_auto_aws_gcp | Mount block devices as individual disks +- name: disks_auto_aws_gcp_azure | Mount block devices as individual disks block: - - name: disks_auto_aws_gcp | auto_vols + - name: disks_auto_aws_gcp_azure | auto_vols debug: msg={{ auto_vols }} - - name: disks_auto_aws_gcp | Get the block device information (pre-filesystem create) + - name: disks_auto_aws_gcp_azure | Get the block device information (pre-filesystem create) blockdevmap: cloud_type: "{{cluster_vars.type}}" become: yes register: r__blockdevmap - - name: disks_auto_aws_gcp | r__blockdevmap (pre-filesystem create) + - name: disks_auto_aws_gcp_azure | r__blockdevmap (pre-filesystem create) debug: msg={{r__blockdevmap}} - - name: disks_auto_aws_gcp | Create filesystem (partitionless) + - name: disks_auto_aws_gcp_azure | Create filesystem (partitionless) become: yes filesystem: fstype: "{{ item.fstype }}" @@ -27,16 +27,16 @@ _dev: "{{ r__blockdevmap.device_map | json_query(\"[?device_name_cloud == '\" + item.device_name + \"' && TYPE=='disk' && parttable_type=='' && FSTYPE=='' && MOUNTPOINT==''].device_name_os | [0]\") }}" when: _dev is defined and _dev != '' - - name: disks_auto_aws_gcp | Get the block device information (post-filesystem create), to get the block IDs for mounting + - name: disks_auto_aws_gcp_azure | Get the block device information (post-filesystem create), to get the block IDs for mounting blockdevmap: cloud_type: "{{cluster_vars.type}}" become: yes register: r__blockdevmap - - name: disks_auto_aws_gcp | r__blockdevmap (post-filesystem create) + - name: disks_auto_aws_gcp_azure | r__blockdevmap (post-filesystem create) debug: msg={{r__blockdevmap}} - - name: disks_auto_aws_gcp | Mount created filesytem(s) persistently + - name: disks_auto_aws_gcp_azure | Mount created filesytem(s) persistently become: yes mount: path: "{{ item.mountpoint }}" @@ -49,7 +49,7 @@ _UUID: "{{ r__blockdevmap.device_map | json_query(\"[?device_name_cloud == '\" + item.device_name + \"' && TYPE=='disk' && parttable_type=='' && MOUNTPOINT==''].UUID | [0]\") }}" when: _UUID is defined and _UUID != '' - - name: disks_auto_aws_gcp | change ownership of mountpoint (if set) + - name: disks_auto_aws_gcp_azure | change ownership of mountpoint (if set) become: yes file: path: "{{ item.mountpoint }}" @@ -59,16 +59,16 @@ group: "{{ item.perms.group | default(omit)}}" loop: "{{auto_vols}}" - - name: disks_auto_aws_gcp | Check that we haven't mounted disks in the wrong place. Especially useful for redeploys when we're moving disks. + - name: disks_auto_aws_gcp_azure | Check that we haven't mounted disks in the wrong place. Especially useful for redeploys when we're moving disks. block: - - name: "disks_auto_aws_gcp | Touch a file with the mountpoint and device name for testing that disk attachment is correct. Note: Use a unique filename here instead of writing to a file, so that more than one file per device is an error." + - name: "disks_auto_aws_gcp_azure | Touch a file with the mountpoint and device name for testing that disk attachment is correct. Note: Use a unique filename here instead of writing to a file, so that more than one file per device is an error." become: yes file: path: "{{item.mountpoint}}/.clusterversetest__{{inventory_hostname | regex_replace('-(?!.*-).*')}}__{{ item.mountpoint | regex_replace('\\/', '_') }}__{{ item.device_name | regex_replace('\/', '_') }}" state: touch loop: "{{auto_vols}}" - - name: disks_auto_aws_gcp | Find all .clusterversetest__ files in mounted disks + - name: disks_auto_aws_gcp_azure | Find all .clusterversetest__ files in mounted disks find: paths: "{{item.mountpoint}}" hidden: yes @@ -76,12 +76,12 @@ loop: "{{auto_vols}}" register: r__find_test - - name: disks_auto_aws_gcp | Check that there is only one .clusterversetest__ file per device in mounted disks. + - name: disks_auto_aws_gcp_azure | Check that there is only one .clusterversetest__ file per device in mounted disks. block: - - name: disks_auto_aws_gcp | testdevicedescriptor + - name: disks_auto_aws_gcp_azure | testdevicedescriptor debug: msg={{testdevicedescriptor}} - - name: disks_auto_aws_gcp | assert that only one device descriptor file exists per disk (otherwise, indicates that this run has mapped either more than one device per mount, or a different one to previous) + - name: disks_auto_aws_gcp_azure | assert that only one device descriptor file exists per disk (otherwise, indicates that this run has mapped either more than one device per mount, or a different one to previous) assert: { that: "testdevicedescriptor | json_query(\"[?length(files) > `1`]\") | length == 0", fail_msg: "ERROR - only a single file should exist per storage device. In error [{{testdevicedescriptor | json_query(\"[?length(files) > `1`]\")}}]" } vars: testdevicedescriptor: "{{ r__find_test | json_query(\"results[].{hostname: '\" + inventory_hostname + \"', device_name: item.device_name, mountpoint: item.mountpoint, files: files[].path}\") }}" @@ -92,27 +92,27 @@ # The following block mounts all attached volumes that have a single, common mountpoint, by creating a logical volume -- name: disks_auto_aws_gcp/lvm | Mount block devices in a single LVM mountpoint through LV/VG +- name: disks_auto_aws_gcp_azure/lvm | Mount block devices in a single LVM mountpoint through LV/VG block: - - name: disks_auto_aws_gcp/lvm | hosttype_vars + - name: disks_auto_aws_gcp_azure/lvm | hosttype_vars debug: msg={{ hosttype_vars }} - - name: disks_auto_aws_gcp/lvm | Install logical volume management tooling. (yum - RedHat/CentOS) + - name: disks_auto_aws_gcp_azure/lvm | Install logical volume management tooling. (yum - RedHat/CentOS) become: true yum: name: "lvm*" state: present when: ansible_os_family == 'RedHat' - - name: disks_auto_aws_gcp/lvm | Get the device information (pre-filesystem create) + - name: disks_auto_aws_gcp_azure/lvm | Get the device information (pre-filesystem create) blockdevmap: become: yes register: r__blockdevmap - - name: disks_auto_aws_gcp/lvm | r__blockdevmap (pre-filesystem create) + - name: disks_auto_aws_gcp_azure/lvm | r__blockdevmap (pre-filesystem create) debug: msg={{r__blockdevmap}} - - name: disks_auto_aws_gcp/lvm | Create a volume group from all block devices + - name: disks_auto_aws_gcp_azure/lvm | Create a volume group from all block devices become: yes lvg: vg: "{{ hosttype_vars.lvmparams.vg_name }}" @@ -120,21 +120,21 @@ vars: auto_vol_device_names: "{{hosttype_vars.auto_volumes | map(attribute='device_name') | sort | join(',')}}" - - name: disks_auto_aws_gcp/lvm | Create a logical volume from volume group + - name: disks_auto_aws_gcp_azure/lvm | Create a logical volume from volume group become: yes lvol: vg: "{{ hosttype_vars.lvmparams.vg_name }}" lv: "{{ hosttype_vars.lvmparams.lv_name }}" size: "{{ hosttype_vars.lvmparams.lv_size }}" - - name: disks_auto_aws_gcp/lvm | Create filesystem(s) on attached volume(s) + - name: disks_auto_aws_gcp_azure/lvm | Create filesystem(s) on attached volume(s) become: yes filesystem: fstype: "{{ hosttype_vars.auto_volumes[0].fstype }}" dev: "/dev/{{ hosttype_vars.lvmparams.vg_name }}/{{ hosttype_vars.lvmparams.lv_name }}" force: no - - name: disks_auto_aws_gcp/lvm | Mount created filesytem(s) persistently + - name: disks_auto_aws_gcp_azure/lvm | Mount created filesytem(s) persistently become: yes mount: path: "{{ hosttype_vars.auto_volumes[0].mountpoint }}" @@ -143,27 +143,27 @@ state: mounted opts: _netdev - - name: disks_auto_aws_gcp/lvm | Check that we haven't mounted disks in the wrong place. Especially useful for redeploys when we're moving disks. + - name: disks_auto_aws_gcp_azure/lvm | Check that we haven't mounted disks in the wrong place. Especially useful for redeploys when we're moving disks. block: - - name: "disks_auto_aws_gcp/lvm | Touch a file with the mountpoint for testing that disk attachment is correct. Note: Use a unique filename here instead of writing to a file, so that more than one file per device is an error." + - name: "disks_auto_aws_gcp_azure/lvm | Touch a file with the mountpoint for testing that disk attachment is correct. Note: Use a unique filename here instead of writing to a file, so that more than one file per device is an error." become: yes file: path: "{{ hosttype_vars.auto_volumes[0].mountpoint }}/.clusterversetest__{{inventory_hostname | regex_replace('-(?!.*-).*')}}__{{ hosttype_vars.auto_volumes[0].mountpoint | regex_replace('\\/', '_') }}" state: touch - - name: disks_auto_aws_gcp/lvm | Find all .clusterversetest__ files in mounted disks + - name: disks_auto_aws_gcp_azure/lvm | Find all .clusterversetest__ files in mounted disks find: paths: "{{ hosttype_vars.auto_volumes[0].mountpoint }}" hidden: yes patterns: ".clusterversetest__*" register: r__find_test - - name: disks_auto_aws_gcp/lvm | Check that there is only one .clusterversetest__ file per device in mounted disks. + - name: disks_auto_aws_gcp_azure/lvm | Check that there is only one .clusterversetest__ file per device in mounted disks. block: - - name: disks_auto_aws_gcp/lvm | testdevicedescriptor + - name: disks_auto_aws_gcp_azure/lvm | testdevicedescriptor debug: msg={{testdevicedescriptor}} - - name: disks_auto_aws_gcp/lvm | assert that only one device descriptor file exists per disk (otherwise, indicates that this run has mapped either more than one device per mount, or a different one to previous) + - name: disks_auto_aws_gcp_azure/lvm | assert that only one device descriptor file exists per disk (otherwise, indicates that this run has mapped either more than one device per mount, or a different one to previous) assert: { that: "testdevicedescriptor | json_query(\"[?length(files) > `1`]\") | length == 0", fail_msg: "ERROR - only a single file should exist per storage device. In error [{{testdevicedescriptor | json_query(\"[?length(files) > `1`]\")}}]" } vars: testdevicedescriptor: "{{ r__find_test | json_query(\"results[].{hostname: '\" + inventory_hostname + \"', device_name: item.device_name, mountpoint: item.mountpoint, files: files[].path}\") }}" diff --git a/config/tasks/main.yml b/config/tasks/main.yml index b99cc683..1aab6595 100644 --- a/config/tasks/main.yml +++ b/config/tasks/main.yml @@ -52,13 +52,13 @@ mode: 0755 when: (static_journal is defined and static_journal|bool) -- name: Create partition table, format and attach volumes - AWS or GCP - include_tasks: disks_auto_aws_gcp.yml - when: cluster_vars.type == "aws" or cluster_vars.type == "gcp" +- name: Create partition table, format and attach volumes - AWS, GCP or Azure + include_tasks: disks_auto_aws_gcp_azure.yml + when: cluster_vars.type == "aws" or cluster_vars.type == "gcp" or cluster_vars.type == "azure" - name: Create partition table, format and attach volumes - generic include_tasks: disks_auto_generic.yml - when: cluster_vars.type != "aws" and cluster_vars.type != "gcp" + when: cluster_vars.type != "aws" and cluster_vars.type != "gcp" and cluster_vars.type != "azure" - name: install prometheus node exporter daemon include_tasks: prometheus_node_exporter.yml diff --git a/create/tasks/azure.yml b/create/tasks/azure.yml new file mode 100644 index 00000000..6f9cdae4 --- /dev/null +++ b/create/tasks/azure.yml @@ -0,0 +1,194 @@ +--- + +- name: cluster_hosts_target_denormalised_by_volume + debug: msg="{{cluster_hosts_target_denormalised_by_volume}}" + +#- name: create/azure | Create storage account (must be [a-z0-9] and <= 24 chars). NOT NECESSARY for IaaS block storage +# azure.azcollection.azure_rm_storageaccount: +# client_id: "{{cluster_vars[buildenv].azure_client_id}}" +# secret: "{{cluster_vars[buildenv].azure_secret}}" +# subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" +# tenant: "{{cluster_vars[buildenv].azure_tenant}}" +# resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" +# name: "{{ (cluster_name | regex_replace('[^a-z0-9]', ''))[:24] }}" +## name: "{{ cluster_suffix }}" +## name: "{{ (item.hostname|hash('md5'))[:24] }}" +# account_type: Standard_LRS +# register: r__azure_rm_storageaccount + + +#### NOTE: +# - Normally, to create an Azure VM, we would create a security group (azure_rm_securitygroup) and if needed, a public IP address (azure_rm_publicipaddress), then attach them +# to a NIC (azure_rm_networkinterface). We would pass this NIC to the VM creation plugin (azure_rm_virtualmachine) in the network_interface_names parameter. +# - Unfortunately, the azure_rm_publicipaddress and azure_rm_networkinterface are not Availability-Zone aware, so when we create the VM (in a specific AZ), the IP is not in +# that zone, so the build fails. +# - The alternative is to build a VM without network_interface_names set. This causes the VM to be built with default public IP and security groups, so we need to change them +# afterwards instead. +#### + +#- name: create/azure | Create security groups +# azure.azcollection.azure_rm_securitygroup: +# client_id: "{{cluster_vars[buildenv].azure_client_id}}" +# secret: "{{cluster_vars[buildenv].azure_secret}}" +# subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" +# tenant: "{{cluster_vars[buildenv].azure_tenant}}" +# resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" +# name: "{{ cluster_name }}" +# tags: +# env: "{{ buildenv }}" +# rules: "{{ cluster_vars.rules }}" +# register: r__azure_rm_securitygroup +# when: cluster_vars.rules | length > 0 +# +#- name: create/azure | r__azure_rm_securitygroup +# debug: msg={{r__azure_rm_securitygroup}} + +#- name: create/azure | Create a public ip address +# azure.azcollection.azure_rm_publicipaddress: +# client_id: "{{cluster_vars[buildenv].azure_client_id}}" +# secret: "{{cluster_vars[buildenv].azure_secret}}" +# subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" +# tenant: "{{cluster_vars[buildenv].azure_tenant}}" +# resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" +# name: "{{item.hostname}}" +# allocation_method: static +## zones: ["{{item.az_name}}"] +# register: r__azure_rm_publicipaddress +# loop: "{{ cluster_hosts_target }}" +# +#- name: create/azure | r__azure_rm_publicipaddress +# debug: msg={{r__azure_rm_publicipaddress}} + +#- name: Create NIC +# azure_rm_networkinterface: +# client_id: "{{cluster_vars[buildenv].azure_client_id}}" +# secret: "{{cluster_vars[buildenv].azure_secret}}" +# subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" +# tenant: "{{cluster_vars[buildenv].azure_tenant}}" +# resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" +# name: "{{item.hostname}}" +# virtual_network: "{{cluster_vars[buildenv].vnet_name}}" +# subnet: "{{cluster_vars[buildenv].vpc_subnet_name_prefix}}" +# ip_configurations: +# - name: "{{item.hostname}}-config" +# public_ip_address_name: "{{item.hostname}}-publicip" +# primary: True +# security_group: "{{r__azure_rm_securitygroup.state.name}}" +# register: r__azure_rm_networkinterface +# loop: "{{ cluster_hosts_target }}" +# +#- name: create/azure | r__azure_rm_networkinterface +# debug: msg={{r__azure_rm_networkinterface}} + + +- name: create/azure | Create VMs asynchronously and wait for completion + block: + - name: create/azure | Detach volumes from previous instances (during the _scheme_rmvm_keepdisk_rollback redeploy, we only redeploy one host at a time, and it is already powered off) + azure.azcollection.azure_rm_manageddisk: + client_id: "{{cluster_vars[buildenv].azure_client_id}}" + secret: "{{cluster_vars[buildenv].azure_secret}}" + subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" + tenant: "{{cluster_vars[buildenv].azure_tenant}}" + resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" + name: "{{item.auto_volume.src.volume_id | basename}}" + managed_by: '' + loop: "{{ cluster_hosts_target_denormalised_by_volume | selectattr('auto_volume.src', 'defined') | list }}" + + - name: create/azure | Create VMs asynchronously + azure.azcollection.azure_rm_virtualmachine: + client_id: "{{cluster_vars[buildenv].azure_client_id}}" + secret: "{{cluster_vars[buildenv].azure_secret}}" + subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" + tenant: "{{cluster_vars[buildenv].azure_tenant}}" + resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" + admin_username: "{{cluster_vars[buildenv].ssh_connection_cfg.host.ansible_user}}" + custom_data : "{{cluster_vars.user_data | default(omit)}}" + image: "{{cluster_vars.image}}" + managed_disk_type: Standard_LRS + name: "{{item.hostname}}" + os_disk_size_gb: "{{item.os_disk_size_gb | default(omit)}}" +# network_interface_names: "{{r__azure_rm_networkinterface.results | json_query(\"[?item.hostname == `\" + item.hostname + \"`].state.name\") }}" + open_ports: ["9"] # tcp/9 is the 'discard' (dev/null) port. It is set because we must put a value in here, otherwise the default tcp/22 is opened to any/any. azure_rm_securitygroup is set below. + public_ip_allocation_method: "{%- if cluster_vars.assign_public_ip == 'yes' -%}Static{%- else -%}Disabled{%- endif -%}" + ssh_password_enabled: no + ssh_public_keys: + - path: "/home/{{cluster_vars[buildenv].ssh_connection_cfg.host.ansible_user}}/.ssh/authorized_keys" + #The ssh key is either provided on the command line (as 'ansible_ssh_private_key_file'), or as a variable in cluster_vars[buildenv].ssh_connection_cfg.host.ansible_ssh_private_key_file (anchored to _host_ssh_connection_cfg.ansible_ssh_private_key_file); we can slurp the key from either variable, and then ssh-keygen it into the public key (we have to remove the comment though before we add our own, (hence the regex), because this is what gcp expects). + key_data: "{%- if _host_ssh_connection_cfg.ansible_ssh_private_key_file is defined -%}{{ lookup('pipe', 'ssh-keygen -y -f /dev/stdin < 0 + +# - name: create/azure | r__azure_rm_securitygroup +# debug: msg={{r__azure_rm_securitygroup}} + + + - name: create/azure | Create and attach managed disk(s) to VM. Do NOT ATTEMPT to do this asynchronously - causes issues! + azure.azcollection.azure_rm_manageddisk: + client_id: "{{cluster_vars[buildenv].azure_client_id}}" + secret: "{{cluster_vars[buildenv].azure_secret}}" + subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" + tenant: "{{cluster_vars[buildenv].azure_tenant}}" + resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" + attach_caching: read_only + disk_size_gb: "{{item.auto_volume.disk_size_gb}}" + lun: "{{item.auto_volume.device_name}}" + managed_by: "{{item.hostname}}" + name: "{{_tags.name}}" + storage_account_type: "{{item.auto_volume.storage_account_type}}" + tags: "{{ _tags | combine(cluster_vars.custom_tagslabels | default({})) }}" + zone: "{{item.az_name}}" + vars: + _tags: + name: "{{item.hostname}}--{{ item.auto_volume.mountpoint | basename }}" + inv_node_version: "{{cluster_vars[buildenv].hosttype_vars[item.hosttype].version | default(omit)}}" + inv_node_type: "{{item.hosttype}}" + owner: "{{ lookup('env','USER') | lower }}" + release: "{{ release_version }}" + loop: "{{ cluster_hosts_target_denormalised_by_volume }}" + register: r__azure_rm_manageddisk + +# - name: create/azure | r__azure_rm_manageddisk +# debug: msg={{r__azure_rm_manageddisk}} diff --git a/create/tasks/esxifree.yml b/create/tasks/esxifree.yml new file mode 100644 index 00000000..ecf54c89 --- /dev/null +++ b/create/tasks/esxifree.yml @@ -0,0 +1,39 @@ +--- + +- name: create/esxifree | Create vmware instances from template + esxifree_guest: + hostname: "{{ cluster_vars.esxi_ip }}" + username: "{{ cluster_vars.username }}" + password: "{{ cluster_vars.password }}" + datastore: "{{ cluster_vars.datastore }}" + template: "{{ cluster_vars.image }}" + name: "{{ item.hostname }}" + state: present + hardware: "{{ {'version': cluster_vars.hardware_version} | combine({'num_cpus': item.flavor['num_cpus'], 'memory_mb': item.flavor['memory_mb']}) }}" + annotation: + Name: "{{item.hostname}}" + hosttype: "{{item.hosttype}}" + env: "{{buildenv}}" + cluster_name: "{{cluster_name}}" + owner: "{{lookup('env','USER')}}" + cluster_suffix: "{{cluster_suffix}}" + lifecycle_state: "current" + cloudinit_userdata: "{{ cluster_vars.cloudinit_userdata | default([]) }}" + disks: "{{ item.auto_volumes | json_query(\"[].{size_gb: volume_size, type: provisioning_type, volname: volname, src: src }\") | default([]) }}" + networks: "{{ cluster_vars[buildenv].networks | default([]) }}" + wait: true + register: esxi_instances + run_once: true + with_items: "{{ cluster_hosts_target }}" + async: 7200 + poll: 0 + +- name: create/esxifree | Wait for instance creation to complete + async_status: + jid: "{{ item.ansible_job_id }}" + register: esxi_jobs + until: esxi_jobs.finished + retries: 300 + with_items: "{{ esxi_instances.results }}" + +#- debug: msg={{esxi_jobs.results}} diff --git a/jenkinsfiles/Jenkinsfile_testsuite b/jenkinsfiles/Jenkinsfile_testsuite index 26b6fab0..d148d9b5 100644 --- a/jenkinsfiles/Jenkinsfile_testsuite +++ b/jenkinsfiles/Jenkinsfile_testsuite @@ -1,25 +1,28 @@ #!groovy +//@Library('MatrixBuilder') +//import org.dougalseeley.MatrixBuilder + + // NOTE: Clusterverse is an independent Ansible role - it cannot run on its own, and needs cluster variables and top-level playbooks of the kind defined in roles/clusterverse/EXAMPLE/ that will import clusterverse // This test suite executes (a matrix of) a series of deploy/redeploy/clean steps that are defined in a Jenkinsfile that are located in such a playbook that can actually run clusterverse (e.g. roles/clusterverse/EXAMPLE/jenkinsfiles/Jenkinsfile_ops) //This will not be needed if we're running this as a multibranch pipeline SCM job, as these are automatically added to the 'scm' variable, but if we instead just cut & paste this file into a pipeline job, they will be used as fallback -def DEFAULT_CLUSTERVERSE_URL = "https://github.com/sky-uk/clusterverse" -def DEFAULT_CLUSTERVERSE_BRANCH = "master" +def DEFAULT_CLUSTERVERSE_URL = "https://github.com/dseeley/clusterverse" +def DEFAULT_CLUSTERVERSE_BRANCH = "dps_add_azure" //Set the git branch for clusterverse_ops to either the PR branch (env.CHANGE_BRANCH), or the current SCM branch (env.BRANCH_NAME) def CV_OPS_GIT_BRANCH = DEFAULT_CLUSTERVERSE_BRANCH if (env.CHANGE_BRANCH) { - CV_OPS_GIT_BRANCH = env.CHANGE_BRANCH -} -else if (env.BRANCH_NAME) { - CV_OPS_GIT_BRANCH = env.BRANCH_NAME + CV_OPS_GIT_BRANCH = env.CHANGE_BRANCH +} else if (env.BRANCH_NAME) { + CV_OPS_GIT_BRANCH = env.BRANCH_NAME } //This allows us to copy&paste this entire script into a pipeline job in the GUI for faster development time (don't have to commit/push to Git to test every change). def scmVars = null if (currentBuild.getBuildCauses('hudson.model.Cause$SCMTriggerCause').size() > 0) { - scmVars = checkout scm + scmVars = checkout scm } @@ -29,73 +32,74 @@ if (currentBuild.getBuildCauses('hudson.model.Cause$SCMTriggerCause').size() > 0 // It takes some inspiration from this blog: https://www.jenkins.io/blog/2019/12/02/matrix-building-with-scripted-pipeline/ /**************************************************************************************/ class MatrixBuilder { - private HashMap jenkinsParamsCopy - private HashMap _matrixParams //This cannot be made into a Closure due to CPS (again). (https://www.jenkins.io/doc/book/pipeline/cps-method-mismatches/) - private Closure clJenkinsParamsMutate - private Closure clMatrixAxesFilter - private Closure clTaskMap - - // NOTE: No constructor. - // When undeclared, constructors are created automatically, creating the instance variables defined above, (where they correspond to the Map that is passed with the instantiation). You can't do a lot of work in a Jenkins Groovy constructor anyway because of CPS (https://www.jenkins.io/doc/book/pipeline/cps-method-mismatches/) - - public Map getTaskMap() { - HashMap tasks = [failFast: false] - _getMatrixAxes().each() { axis -> - List axisEnvVars = axis.collect { key, val -> "${key}=${val}" } - axisEnvVars.add("BUILD_HASH=" + generateMD5(hashCode() + axisEnvVars.join(','), 12)) //A unique build hash of the classid (hashcode) and the matrix elements - tasks[axisEnvVars.join(', ')] = { this.clTaskMap(axisEnvVars) } + private HashMap jenkinsParams + private HashMap _matrixParams //This cannot be made into a Closure due to CPS (again). (https://www.jenkins.io/doc/book/pipeline/cps-method-mismatches/) + private Closure clJenkinsParamsMutate //A closure that mutates the Jenkins params _before_ we calculate the axes. Useful in case some params are not intended to be part of the axes (and should be removed) + private Closure clMatrixAxesFilter //A closure of invalid axis combinations, which allows us to filter out combinations that are incompatible with each other (e.g. testing internet explorer on linux) + private Closure clTaskMap + + // NOTE: No constructor. + // When undeclared, constructors are created automatically, creating the instance variables defined above, (where they correspond to the Map that is passed with the instantiation). You can't do a lot of work in a Jenkins Groovy constructor anyway because of CPS (https://www.jenkins.io/doc/book/pipeline/cps-method-mismatches/) + + public Map getTaskMap() { + HashMap tasks = [failFast: false] + _getMatrixAxes().each() { axis -> + List axisEnvVars = axis.collect { key, val -> "${key}=${val}" } + axisEnvVars.add("BUILD_HASH=" + generateMD5(hashCode() + axisEnvVars.join(','), 12)) //A unique build hash of the classid (hashcode) and the matrix elements + tasks[axisEnvVars.join(', ')] = { this.clTaskMap(axisEnvVars) } + } + return (tasks) + } + + private List _getMatrixAxes() { + this._getMatrixParams() + List allCombinations = this._getAxesCombinations() + return (this.clMatrixAxesFilter ? allCombinations.findAll(this.clMatrixAxesFilter) : allCombinations) + } + + private HashMap _getMatrixParams() { + HashMap newMatrixParams = Eval.me(this.jenkinsParams.inspect()) + newMatrixParams = this.clJenkinsParamsMutate ? this.clJenkinsParamsMutate(newMatrixParams) : newMatrixParams + newMatrixParams = newMatrixParams.each { key, choice -> newMatrixParams.put(key, (choice instanceof String) ? choice.split(',') : choice.toString()) } //newMatrixParams().each { param -> param.value = (param.value instanceof String) ? param.value.split(',') : param.value } //NOTE: Doesn't work: https://www.jenkins.io/doc/book/pipeline/cps-method-mismatches/ + this._matrixParams = newMatrixParams + return (newMatrixParams) } - return (tasks) - } - - private List _getMatrixAxes() { - this._getMatrixParams() - List allCombinations = this._getAxesCombinations() - return (this.clMatrixAxesFilter ? allCombinations.findAll(this.clMatrixAxesFilter) : allCombinations) - } - - private HashMap _getMatrixParams() { - HashMap newMatrixParams = Eval.me(this.jenkinsParamsCopy.inspect()) - newMatrixParams = this.clJenkinsParamsMutate ? this.clJenkinsParamsMutate(newMatrixParams) : newMatrixParams - newMatrixParams = newMatrixParams.each { key, choice -> newMatrixParams.put(key, (choice instanceof String) ? choice.split(',') : choice.toString()) } //newMatrixParams().each { param -> param.value = (param.value instanceof String) ? param.value.split(',') : param.value } //NOTE: Doesn't work: https://www.jenkins.io/doc/book/pipeline/cps-method-mismatches/ - this._matrixParams = newMatrixParams - return (newMatrixParams) - } - - @NonCPS - private List _getAxesCombinations() { - List axes = [] - this._matrixParams.each { axis, values -> - List axisList = [] - values.each { value -> - axisList << [(axis): value] - } - axes << axisList + + @NonCPS + private List _getAxesCombinations() { + List axes = [] + this._matrixParams.each { axis, values -> + List axisList = [] + values.each { value -> + axisList << [(axis): value] + } + axes << axisList + } + axes.combinations()*.sum() // calculates the cartesian product } - axes.combinations()*.sum() // calculates the cartesian product - } - static String generateMD5(String s, int len = 31) { - java.security.MessageDigest.getInstance("MD5").digest(s.bytes).encodeHex().toString()[0..len] - } + static String generateMD5(String s, int len = 31) { + java.security.MessageDigest.getInstance("MD5").digest(s.bytes).encodeHex().toString()[0..len] + } } properties([ - //disableConcurrentBuilds(), - //pipelineTriggers([pollSCM(ignorePostCommitHooks: true, scmpoll_spec: '''H/30 8-19 * * 1-5''')]), - parameters([ - extendedChoice(name: 'CLOUD_REGION', type: 'PT_MULTI_SELECT', value: 'aws/us-west-2,aws/eu-central-1,aws/eu-west-1,gcp/us-west2,gcp/europe-west1', description: 'Specify which cloud/region(s) to test', visibleItemCount: 5), - choice(name: 'BUILDENV', choices: ['', 'dev', 'mgmt'], description: "The environment in which to run the tests"), - string(name: 'CLUSTER_ID', defaultValue: 'top_peacock', trim: true), - [name: 'DNS_FORCE_DISABLE', $class: 'ChoiceParameter', choiceType: 'PT_RADIO', description: '', randomName: 'choice-parameter-31196915540455', script: [$class: 'GroovyScript', fallbackScript: [classpath: [], sandbox: true, script: ''], script: [classpath: [], sandbox: true, script: 'return [\'false:selected\',\'true\',\'true,false\']']]], - extendedChoice(name: 'REDEPLOY_SCHEME', type: 'PT_CHECKBOX', value: '_scheme_addallnew_rmdisk_rollback,_scheme_addnewvm_rmdisk_rollback,_scheme_rmvm_rmdisk_only,_scheme_rmvm_keepdisk_rollback', defaultValue: '_scheme_addallnew_rmdisk_rollback,_scheme_addnewvm_rmdisk_rollback,_scheme_rmvm_rmdisk_only,_scheme_rmvm_keepdisk_rollback', description: 'Specify which redeploy scheme(s) to test', visibleItemCount: 5), - choice(name: 'CLEAN_ON_FAILURE', choices: [true, false], description: "Run a clusterverse clean in the event of a failure."), - extendedChoice(name: 'MYHOSTTYPES_TEST', type: 'PT_MULTI_SELECT', value: 'nomyhosttypes,myhosttypes', defaultValue: 'nomyhosttypes', descriptionPropertyValue: 'Without myhosttypes, With myhosttypes', description: 'Whether to run tests on pre-configured hosttypes.', visibleItemCount: 2), - [name: 'MYHOSTTYPES_LIST', $class: 'DynamicReferenceParameter', choiceType: 'ET_FORMATTED_HTML', description: 'These hosttype definitions must exist in cluster_vars for all clusters', randomName: 'choice-parameter-423779762617532', referencedParameters: 'MYHOSTTYPES_TEST', script: [$class: 'GroovyScript', fallbackScript: [classpath: [], sandbox: true, script: 'return ""'], script: [classpath: [], sandbox: true, script: 'if (MYHOSTTYPES_TEST.split(\',\').contains(\'myhosttypes\')) { return ("") }']]], - [name: 'MYHOSTTYPES_SERIAL_PARALLEL', $class: 'CascadeChoiceParameter', choiceType: 'PT_RADIO', description: 'Run the myhosttype test in serial or parallel', randomName: 'choice-parameter-424489601389882', referencedParameters: 'MYHOSTTYPES_TEST', script: [$class: 'GroovyScript', fallbackScript: [classpath: [], sandbox: true, script: 'return([])'], script: [classpath: [], sandbox: true, script: 'if (MYHOSTTYPES_TEST==\'nomyhosttypes,myhosttypes\') { return([\'serial:selected\',\'parallel\']) }']]], - extendedChoice(name: 'IMAGE_TESTED', type: 'PT_MULTI_SELECT', value: '_ubuntu2004image,_centos7image', defaultValue: '_ubuntu2004image', descriptionPropertyValue: 'Ubuntu 20.04, CentOS 7', description: 'Specify which image(s) to test', visibleItemCount: 3), - ]) + //disableConcurrentBuilds(), + //pipelineTriggers([pollSCM(ignorePostCommitHooks: true, scmpoll_spec: '''H/30 8-19 * * 1-5''')]), + parameters([ + extendedChoice(name: 'CLOUD_REGION', type: 'PT_MULTI_SELECT', value: 'esxifree/dougalab,aws/eu-west-1,gcp/europe-west1,azure/westeurope', description: 'Specify which cloud/region(s) to test', visibleItemCount: 5), + choice(name: 'BUILDENV', choices: ['', 'dev'], description: "The environment in which to run the tests"), + string(name: 'CLUSTER_ID', defaultValue: 'testsuite', trim: true), + [name: 'DNS_FORCE_DISABLE', $class: 'ChoiceParameter', choiceType: 'PT_RADIO', description: '', randomName: 'choice-parameter-31196915540455', script: [$class: 'GroovyScript', fallbackScript: [classpath: [], sandbox: true, script: ''], script: [classpath: [], sandbox: true, script: 'return [\'false:selected\',\'true\',\'true,false\']']]], + extendedChoice(name: 'REDEPLOY_SCHEME', type: 'PT_CHECKBOX', value: '_scheme_addallnew_rmdisk_rollback,_scheme_addnewvm_rmdisk_rollback,_scheme_rmvm_rmdisk_only,_scheme_rmvm_keepdisk_rollback', defaultValue: '_scheme_addallnew_rmdisk_rollback,_scheme_addnewvm_rmdisk_rollback,_scheme_rmvm_rmdisk_only,_scheme_rmvm_keepdisk_rollback', description: 'Specify which redeploy scheme(s) to test', visibleItemCount: 5), + choice(name: 'CLEAN_ON_FAILURE', choices: [true, false], description: "Run a clusterverse clean in the event of a failure."), + extendedChoice(name: 'MYHOSTTYPES_TEST', type: 'PT_MULTI_SELECT', value: 'nomyhosttypes,myhosttypes', defaultValue: 'nomyhosttypes', descriptionPropertyValue: 'Without myhosttypes, With myhosttypes', description: 'Whether to run tests on pre-configured hosttypes.', visibleItemCount: 3), + [name: 'MYHOSTTYPES_LIST', $class: 'DynamicReferenceParameter', choiceType: 'ET_FORMATTED_HTML', description: 'These hosttype definitions must exist in cluster_vars for all clusters', randomName: 'choice-parameter-423779762617532', referencedParameters: 'MYHOSTTYPES_TEST', script: [$class: 'GroovyScript', fallbackScript: [classpath: [], sandbox: true, script: 'return ""'], script: [classpath: [], sandbox: true, script: 'if (MYHOSTTYPES_TEST.split(\',\').contains(\'myhosttypes\')) { return ("") }']]], + [name: 'MYHOSTTYPES_SERIAL_PARALLEL', $class: 'CascadeChoiceParameter', choiceType: 'PT_RADIO', description: 'Run the myhosttype test in serial or parallel', randomName: 'choice-parameter-424489601389882', referencedParameters: 'MYHOSTTYPES_TEST', script: [$class: 'GroovyScript', fallbackScript: [classpath: [], sandbox: true, script: 'return([])'], script: [classpath: [], sandbox: true, script: 'if (MYHOSTTYPES_TEST==\'nomyhosttypes,myhosttypes\') { return([\'serial:selected\',\'parallel\']) }']]], + extendedChoice(name: 'SCALEUPDOWN', type: 'PT_MULTI_SELECT', value: 'noscale,scaleup,scaledown', defaultValue: 'noscale', description: 'Specify whether to test scaling up and/or down.', visibleItemCount: 3), + extendedChoice(name: 'IMAGE_TESTED', type: 'PT_MULTI_SELECT', value: '_ubuntu2004image,_centos7image', defaultValue: '_ubuntu2004image', descriptionPropertyValue: 'Ubuntu 20.04, CentOS 7', description: 'Specify which image(s) to test', visibleItemCount: 3), + ]) ]) println("User-supplied 'params': \n" + params.inspect() + "\n") @@ -107,155 +111,193 @@ println("User-supplied 'params': \n" + params.inspect() + "\n") // A class to hold the status of each stage, so we can fail a stage and be able to run the clean at the end if needed class cStageBuild { - public String result = 'SUCCESS' - public HashMap userParams = [:] - - String getUserParamsString() { - String userParamsString = "" - this.userParams.each({paramName, paramVal -> - userParamsString += " -e ${paramName}=${paramVal}" - }) - return(userParamsString + " -vvvv") - } + public String result = 'SUCCESS' + public HashMap userParams = [:] + + String getUserParamsString() { + String userParamsString = "" + this.userParams.each({ paramName, paramVal -> + userParamsString += " -e ${paramName}=${paramVal}" + }) + return (userParamsString + " -vvvv") + } } // A pipeline 'stage' template for clusterverse-ops boilerplate def stage_cvops(String stageLabel, cStageBuild stageBuild, Closure stageExpressions) { - stage(stageLabel) { - if (stageBuild.result == 'SUCCESS') { - try { - stageExpressions() - } catch (Exception err) { - currentBuild.result = 'FAILURE' - stageBuild.result = 'FAILURE' - unstable('Stage failed! Error was: ' + err) // OR: 'error "Stage failure"' or 'throw new org.jenkinsci.plugins.workflow.steps.FlowInterruptedException(hudson.model.Result.FAILURE)', but both of these fail all future stages, preventing us calling the clean. - } + stage(stageLabel) { + if (stageBuild.result == 'SUCCESS') { + try { + stageExpressions() + } catch (Exception err) { + currentBuild.result = 'FAILURE' + stageBuild.result = 'FAILURE' + unstable('Stage failed! Error was: ' + err) // OR: 'error "Stage failure"' or 'throw new org.jenkinsci.plugins.workflow.steps.FlowInterruptedException(hudson.model.Result.FAILURE)', but both of these fail all future stages, preventing us calling the clean. + } + } } - } } -/**************************************************************************************/ -// A 'self-test' matrix. Doesn't actually do anything, just tests the logic of the matrix -/**************************************************************************************/ -SELFTEST = new MatrixBuilder([ - jenkinsParamsCopy: params, - clJenkinsParamsMutate: { jenkinsParamsCopy -> - jenkinsParamsCopy.remove('MYHOSTTYPES_LIST') - jenkinsParamsCopy.remove('MYHOSTTYPES_TEST') - jenkinsParamsCopy.remove('MYHOSTTYPES_SERIAL_PARALLEL') - jenkinsParamsCopy.remove('CLEAN_ON_FAILURE') - return jenkinsParamsCopy - }, - clMatrixAxesFilter: { axis -> - !(params.DNS_TEST == 'both' && axis['DNS_FORCE_DISABLE'] == 'true' && axis['CLOUD_REGION'] == 'esxifree/dougalab') && - !(axis['IMAGE_TESTED'] != '_ubuntu2004image' && axis['CLOUD_REGION'] == 'esxifree/dougalab') - }, - clTaskMap: { axisEnvVars -> - node { - withEnv(axisEnvVars) { - withCredentials([string(credentialsId: "VAULT_PASSWORD_${env.BUILDENV.toUpperCase()}", variable: 'VAULT_PASSWORD_BUILDENV')]) { - env.VAULT_PASSWORD_BUILDENV = VAULT_PASSWORD_BUILDENV - } - sh 'printenv | sort' - def stageBuild = new cStageBuild([result: 'SUCCESS']) - - stage_cvops('deploy', stageBuild, { - echo "deploy" - }) - - stage_cvops('redeploy (1/4 fail)', stageBuild, { - echo "redeploy" - //Test that script can fail individual stages (1 in 4 should fail) - def x = Math.abs(new Random().nextInt() % 4) + 1 - if (x == 1) throw new IllegalStateException("Test failed stage") - }) - - stage_cvops('deploy on top', stageBuild, { - echo "deploy on top" - }) - } - } - } -]) +///**************************************************************************************/ +//// A 'self-test' matrix. Doesn't actually do anything, just tests the logic of the matrix +///**************************************************************************************/ +////SELFTEST = library('MatrixBuilder').org.dougalseeley.MatrixBuilder.new([ +//SELFTEST = new MatrixBuilder([ +// jenkinsParams: params, +// clJenkinsParamsMutate: { jenkinsParams -> +// jenkinsParams.remove('MYHOSTTYPES_LIST') +// jenkinsParams.remove('MYHOSTTYPES_TEST') +// jenkinsParams.remove('MYHOSTTYPES_SERIAL_PARALLEL') +// jenkinsParams.remove('CLEAN_ON_FAILURE') +// return jenkinsParams +// }, +// clMatrixAxesFilter: { axis -> +// !(params.DNS_TEST == 'both' && axis['DNS_FORCE_DISABLE'] == 'true' && axis['CLOUD_REGION'] == 'esxifree/dougalab') && /* DNS is not supported in dougalab */ +// !(axis['IMAGE_TESTED'] != '_ubuntu2004image' && axis['CLOUD_REGION'] == 'esxifree/dougalab') && /* Only _ubuntu2004image is supported in dougalab */ +// !(axis['REDEPLOY_SCHEME'] == '_scheme_rmvm_keepdisk_rollback' && axis['CLOUD_REGION'].split('/')[0] == 'azure') && /* _scheme_rmvm_keepdisk_rollback not supported in Azure */ +// !(axis['REDEPLOY_SCHEME'] == '_scheme_rmvm_rmdisk_only' && axis['SCALEUPDOWN'] == 'scaledown') && /* _scheme_rmvm_rmdisk_only only supports scaling up */ +// !(axis['REDEPLOY_SCHEME'] == '_scheme_rmvm_keepdisk_rollback' && (axis['SCALEUPDOWN'] == 'scaledown' || axis['SCALEUPDOWN'] == 'scaleup')) /* _scheme_rmvm_keepdisk_rollback does not support scaling */ +// }, +// clTaskMap: { axisEnvVars -> +// node { +// withEnv(axisEnvVars) { +// withCredentials([string(credentialsId: "VAULT_PASSWORD_${env.BUILDENV.toUpperCase()}", variable: 'VAULT_PASSWORD_BUILDENV')]) { +// env.VAULT_PASSWORD_BUILDENV = VAULT_PASSWORD_BUILDENV +// } +// sh 'printenv | sort' +// def stageBuild = new cStageBuild([result: 'SUCCESS']) +// +// stage_cvops('deploy', stageBuild, { +// echo "deploy" +// }) +// +// stage_cvops('redeploy (1/4 fail)', stageBuild, { +// echo "redeploy" +// //Test that script can fail individual stages (1 in 4 should fail) +// def x = Math.abs(new Random().nextInt() % 4) + 1 +// if (x == 1) throw new IllegalStateException("Test failed stage") +// }) +// +// stage_cvops('deploy on top', stageBuild, { +// echo "deploy on top" +// }) +// } +// } +// } +//]) /**************************************************************************************/ // Runs tests *without* setting myhosttypes. This is a relatively straightforward application of the matrix algorithm. /**************************************************************************************/ CVTEST_NOMYHOSTTYPES = new MatrixBuilder([ - jenkinsParamsCopy: params, - clJenkinsParamsMutate: { jenkinsParamsCopy -> - jenkinsParamsCopy.remove('MYHOSTTYPES_LIST') - jenkinsParamsCopy.remove('MYHOSTTYPES_TEST') - jenkinsParamsCopy.remove('MYHOSTTYPES_SERIAL_PARALLEL') - jenkinsParamsCopy.remove('CLEAN_ON_FAILURE') - return jenkinsParamsCopy - }, - clMatrixAxesFilter: { axis -> - !(params.DNS_TEST == 'both' && axis['DNS_FORCE_DISABLE'] == 'true' && axis['CLOUD_REGION'] == 'esxifree/dougalab') && - !(axis['IMAGE_TESTED'] != '_ubuntu2004image' && axis['CLOUD_REGION'] == 'esxifree/dougalab') - }, - clTaskMap: { axisEnvVars -> - node { - withEnv(axisEnvVars) { - withCredentials([string(credentialsId: "VAULT_PASSWORD_${env.BUILDENV.toUpperCase()}", variable: 'VAULT_PASSWORD_BUILDENV')]) { - env.VAULT_PASSWORD_BUILDENV = VAULT_PASSWORD_BUILDENV - } - sh 'printenv | sort' - def stageBuild = new cStageBuild([result: 'SUCCESS']) - - if (env.IMAGE_TESTED) { - stageBuild.userParams.put("cluster_vars_override", "\\\'{\\\"image\\\":\\\"{{${env.IMAGE_TESTED}}}\\\"}\\\'") //NOTE: NO SPACES are allowed in this!! - } - - - stageBuild.userParams.put("skip_release_version_check", "true") - stageBuild.userParams.put("release_version", "1_0_0") - stage_cvops('deploy', stageBuild, { - build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'deploy'), string(name: 'REDEPLOY_SCHEME', value: ''), string(name: 'CANARY', value: 'none'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: true), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.userParamsString())] - }) - - if (env.REDEPLOY_SCHEME) { - stageBuild.userParams.put("release_version", "2_0_0") - stage_cvops('redeploy canary=start', stageBuild, { - build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'redeploy'), string(name: 'REDEPLOY_SCHEME', value: (env.REDEPLOY_SCHEME ? env.REDEPLOY_SCHEME : '')), string(name: 'CANARY', value: 'start'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: false), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.userParamsString())] - }) - - stage_cvops('redeploy canary=finish', stageBuild, { - build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'redeploy'), string(name: 'REDEPLOY_SCHEME', value: (env.REDEPLOY_SCHEME ? env.REDEPLOY_SCHEME : '')), string(name: 'CANARY', value: 'finish'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: false), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.userParamsString())] - }) - - stage_cvops('redeploy canary=tidy', stageBuild, { - build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'redeploy'), string(name: 'REDEPLOY_SCHEME', value: (env.REDEPLOY_SCHEME ? env.REDEPLOY_SCHEME : '')), string(name: 'CANARY', value: 'tidy'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: false), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.userParamsString())] - }) - - stageBuild.userParams.put("release_version", "3_0_0") - stage_cvops('redeploy canary=none (tidy_on_success)', stageBuild, { - build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'redeploy'), string(name: 'REDEPLOY_SCHEME', value: (env.REDEPLOY_SCHEME ? env.REDEPLOY_SCHEME : '')), string(name: 'CANARY', value: 'none'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: true), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.userParamsString())] - }) - } else { - stage_cvops('Redeploy not requested', stageBuild, { - echo "Redeploy testing not requested" - }) - } - - stage_cvops('deploy on top', stageBuild, { - build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'deploy'), string(name: 'REDEPLOY_SCHEME', value: ''), string(name: 'CANARY', value: 'none'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: true), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.userParamsString())] - }) - - if (stageBuild.result == 'SUCCESS' || params.CLEAN_ON_FAILURE == 'true') { - stage('clean') { - if (stageBuild.result != 'SUCCESS') { - echo "Stage failure: Running clean-up on cluster..." - } - catchError { - build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'clean'), string(name: 'REDEPLOY_SCHEME', value: ''), string(name: 'CANARY', value: 'none'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: true), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.userParamsString())] - } + jenkinsParams: params, + clJenkinsParamsMutate: { jenkinsParams -> + jenkinsParams.remove('MYHOSTTYPES_LIST') + jenkinsParams.remove('MYHOSTTYPES_TEST') + jenkinsParams.remove('MYHOSTTYPES_SERIAL_PARALLEL') + jenkinsParams.remove('CLEAN_ON_FAILURE') + return jenkinsParams + }, + clMatrixAxesFilter: { axis -> + !(params.DNS_TEST == 'both' && axis['DNS_FORCE_DISABLE'] == 'true' && axis['CLOUD_REGION'] == 'esxifree/dougalab') && /* DNS is not supported in dougalab */ + !(axis['IMAGE_TESTED'] != '_ubuntu2004image' && axis['CLOUD_REGION'] == 'esxifree/dougalab') && /* Only _ubuntu2004image is supported in dougalab */ + !(axis['REDEPLOY_SCHEME'] == '_scheme_rmvm_keepdisk_rollback' && axis['CLOUD_REGION'].split('/')[0] == 'azure') && /* _scheme_rmvm_keepdisk_rollback not supported in Azure */ + !(axis['REDEPLOY_SCHEME'] == '_scheme_rmvm_rmdisk_only' && axis['SCALEUPDOWN'] == 'scaledown') && /* _scheme_rmvm_rmdisk_only only supports scaling up */ + !(axis['REDEPLOY_SCHEME'] == '_scheme_rmvm_keepdisk_rollback' && (axis['SCALEUPDOWN'] == 'scaledown' || axis['SCALEUPDOWN'] == 'scaleup')) /* _scheme_rmvm_keepdisk_rollback does not support scaling */ + }, + clTaskMap: { axisEnvVars -> + node { + withEnv(axisEnvVars) { + withCredentials([string(credentialsId: "VAULT_PASSWORD_${env.BUILDENV.toUpperCase()}", variable: 'VAULT_PASSWORD_BUILDENV')]) { + env.VAULT_PASSWORD_BUILDENV = VAULT_PASSWORD_BUILDENV + } + sh 'printenv | sort' + def stageBuild = new cStageBuild([result: 'SUCCESS']) + HashMap cluster_vars_override = [:] + + if (env.IMAGE_TESTED) { + cluster_vars_override += [image: "{{${env.IMAGE_TESTED}}}"] + stageBuild.userParams.put("cluster_vars_override", "\\\'" + groovy.json.JsonOutput.toJson(cluster_vars_override).replace("\"", "\\\"") + "\\\'") //NOTE: NO SPACES are allowed in this!! + } + + stageBuild.userParams.put("skip_release_version_check", "true") + stageBuild.userParams.put("release_version", "1_0_0") + stage_cvops('deploy', stageBuild, { + build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'deploy'), string(name: 'REDEPLOY_SCHEME', value: ''), string(name: 'CANARY', value: 'none'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: true), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.getUserParamsString())] + }) + + // Update the clustervars with new scaled cluster size + if (env.SCALEUPDOWN == 'scaleup') { + cluster_vars_override += ["${env.BUILDENV}": [hosttype_vars: [sys: [vms_by_az: [b: 1, c: 1]]]]] // AZ 'c' is not set normally + } else if (env.SCALEUPDOWN == 'scaledown') { + cluster_vars_override += ["${env.BUILDENV}": [hosttype_vars: [sys: [vms_by_az: [b: 0, c: 0]]]]] // AZ 'b' is set normally + } + + if (cluster_vars_override.size()) { + stageBuild.userParams.put("cluster_vars_override", "\\\'" + groovy.json.JsonOutput.toJson(cluster_vars_override).replace("\"", "\\\"") + "\\\'") //NOTE: NO SPACES are allowed in this!! + } + + if (env.REDEPLOY_SCHEME) { + stageBuild.userParams.put("release_version", "2_0_0") + stage_cvops('redeploy canary=start', stageBuild, { + build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'redeploy'), string(name: 'REDEPLOY_SCHEME', value: (env.REDEPLOY_SCHEME ? env.REDEPLOY_SCHEME : '')), string(name: 'CANARY', value: 'start'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: false), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.getUserParamsString())] + }) + + stage_cvops('redeploy canary=finish', stageBuild, { + build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'redeploy'), string(name: 'REDEPLOY_SCHEME', value: (env.REDEPLOY_SCHEME ? env.REDEPLOY_SCHEME : '')), string(name: 'CANARY', value: 'finish'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: false), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.getUserParamsString())] + }) + + stage_cvops('redeploy canary=tidy', stageBuild, { + build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'redeploy'), string(name: 'REDEPLOY_SCHEME', value: (env.REDEPLOY_SCHEME ? env.REDEPLOY_SCHEME : '')), string(name: 'CANARY', value: 'tidy'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: false), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.getUserParamsString())] + }) + + //Need to redeploy original cluster (without scaling), to test that scaling works with next redeploy test (canary=none) + if (env.SCALEUPDOWN == 'scaleup' || env.SCALEUPDOWN == 'scaledown') { + cluster_vars_override.remove("${env.BUILDENV}") + stageBuild.userParams.put("cluster_vars_override", "\\\'" + groovy.json.JsonOutput.toJson(cluster_vars_override).replace("\"", "\\\"") + "\\\'") + stageBuild.userParams.put("release_version", "2_5_0") + stage_cvops('deploy clean original (unscaled) for next test', stageBuild, { + build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'deploy'), string(name: 'REDEPLOY_SCHEME', value: ''), string(name: 'CANARY', value: 'none'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: true), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.getUserParamsString() + " -e clean=_all_")] + }) + + //Re-add the scaleup/down cmdline for next redeploy test (canary=none) + if (env.SCALEUPDOWN == 'scaleup') { + cluster_vars_override += ["${env.BUILDENV}": [hosttype_vars: [sys: [vms_by_az: [b: 1, c: 1]]]]] // AZ 'c' is not set normally + } else if (env.SCALEUPDOWN == 'scaledown') { + cluster_vars_override += ["${env.BUILDENV}": [hosttype_vars: [sys: [vms_by_az: [b: 0, c: 0]]]]] // AZ 'b' is set normally + } + stageBuild.userParams.put("cluster_vars_override", "\\\'" + groovy.json.JsonOutput.toJson(cluster_vars_override).replace("\"", "\\\"") + "\\\'") + } + + // Run the canary=none redeploy + stageBuild.userParams.put("release_version", "3_0_0") + stage_cvops('redeploy canary=none (tidy_on_success)', stageBuild, { + build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'redeploy'), string(name: 'REDEPLOY_SCHEME', value: (env.REDEPLOY_SCHEME ? env.REDEPLOY_SCHEME : '')), string(name: 'CANARY', value: 'none'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: true), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.getUserParamsString())] + }) + } else { + stage_cvops('Redeploy not requested', stageBuild, { + echo "Redeploy testing not requested" + }) + } + + stage_cvops('deploy on top', stageBuild, { + build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'deploy'), string(name: 'REDEPLOY_SCHEME', value: ''), string(name: 'CANARY', value: 'none'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: true), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.getUserParamsString())] + }) + + if (stageBuild.result == 'SUCCESS' || params.CLEAN_ON_FAILURE == 'true') { + stage('clean') { + if (stageBuild.result != 'SUCCESS') { + echo "Stage failure: Running clean-up on cluster..." + } + catchError { + build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'clean'), string(name: 'REDEPLOY_SCHEME', value: ''), string(name: 'CANARY', value: 'none'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: true), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.getUserParamsString())] + } + } + } + } } - } } - } - } ]) @@ -264,87 +306,121 @@ CVTEST_NOMYHOSTTYPES = new MatrixBuilder([ // The logic of doing this is different to the matrix without myhosttypes, hence a separate matrix. /**************************************************************************************/ CVTEST_MYHOSTTYPES = new MatrixBuilder([ - jenkinsParamsCopy: params, - clJenkinsParamsMutate: { jenkinsParamsCopy -> - jenkinsParamsCopy.remove('MYHOSTTYPES_LIST') - jenkinsParamsCopy.remove('MYHOSTTYPES_TEST') - jenkinsParamsCopy.remove('MYHOSTTYPES_SERIAL_PARALLEL') - jenkinsParamsCopy.remove('CLEAN_ON_FAILURE') - return jenkinsParamsCopy - }, - clMatrixAxesFilter: { axis -> - !(params.DNS_TEST == 'both' && axis['DNS_FORCE_DISABLE'] == 'true' && axis['CLOUD_REGION'] == 'esxifree/dougalab') && - !(axis['IMAGE_TESTED'] != '_ubuntu2004image' && axis['CLOUD_REGION'] == 'esxifree/dougalab') && - !(axis['REDEPLOY_SCHEME'] == '_scheme_addallnew_rmdisk_rollback') - }, - clTaskMap: { axisEnvVars -> - node { - withEnv(axisEnvVars) { - withCredentials([string(credentialsId: "VAULT_PASSWORD_${env.BUILDENV.toUpperCase()}", variable: 'VAULT_PASSWORD_BUILDENV')]) { - env.VAULT_PASSWORD_BUILDENV = VAULT_PASSWORD_BUILDENV - } - sh 'printenv | sort' - def stageBuild = new cStageBuild([result: 'SUCCESS']) - - if (env.IMAGE_TESTED) { - stageBuild.userParams.put("cluster_vars_override", "\\\'{\\\"image\\\":\\\"{{${env.IMAGE_TESTED}}}\\\"}\\\'") //NOTE: NO SPACES are allowed in this!! - } - - if (env.REDEPLOY_SCHEME) { - if (params.MYHOSTTYPES_LIST == '') { - currentBuild.result = 'FAILURE' - stageBuild.result = 'FAILURE' - unstable('Stage failed! Error was: ' + err) // OR: 'error "Stage failure"' or 'throw new org.jenkinsci.plugins.workflow.steps.FlowInterruptedException(hudson.model.Result.FAILURE)', but both of these fail all future stages, preventing us calling the clean. - } - - stageBuild.userParams.put("skip_release_version_check", "true") - stageBuild.userParams.put("release_version", "1_0_0") - stage_cvops('deploy', stageBuild, { - build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'deploy'), string(name: 'REDEPLOY_SCHEME', value: ''), string(name: 'CANARY', value: 'none'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: true), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.userParamsString())] - }) - - // Run the split redeploy over all hosttypes - stageBuild.userParams.put("release_version", "2_0_0") - params.MYHOSTTYPES_LIST.split(',').each({ my_host_type -> - stage_cvops("redeploy canary=start ($my_host_type)", stageBuild, { - build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'redeploy'), string(name: 'REDEPLOY_SCHEME', value: (env.REDEPLOY_SCHEME ? env.REDEPLOY_SCHEME : '')), string(name: 'CANARY', value: 'start'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: false), string(name: 'MYHOSTTYPES', value: my_host_type), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.userParamsString())] - }) - - stage_cvops("redeploy canary=finish ($my_host_type)", stageBuild, { - build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'redeploy'), string(name: 'REDEPLOY_SCHEME', value: (env.REDEPLOY_SCHEME ? env.REDEPLOY_SCHEME : '')), string(name: 'CANARY', value: 'finish'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: false), string(name: 'MYHOSTTYPES', value: my_host_type), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.userParamsString())] - }) - - stage_cvops("redeploy canary=tidy ($my_host_type)", stageBuild, { - build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'redeploy'), string(name: 'REDEPLOY_SCHEME', value: (env.REDEPLOY_SCHEME ? env.REDEPLOY_SCHEME : '')), string(name: 'CANARY', value: 'tidy'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: false), string(name: 'MYHOSTTYPES', value: my_host_type), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.userParamsString())] - }) - }) - - // Run the mono-redeploy over all hosttypes - stageBuild.userParams.put("release_version", "3_0_0") - params.MYHOSTTYPES_LIST.split(',').each({ my_host_type -> - stage_cvops("redeploy canary=none ($my_host_type) (tidy_on_success)", stageBuild, { - build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'redeploy'), string(name: 'REDEPLOY_SCHEME', value: (env.REDEPLOY_SCHEME ? env.REDEPLOY_SCHEME : '')), string(name: 'CANARY', value: 'none'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: true), string(name: 'MYHOSTTYPES', value: my_host_type), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.userParamsString())] - }) - }) - - if (stageBuild.result == 'SUCCESS' || params.CLEAN_ON_FAILURE == 'true') { - stage('clean') { - if (stageBuild.result != 'SUCCESS') { - echo "Stage failure: Running clean-up on cluster..." + jenkinsParams: params, + clJenkinsParamsMutate: { jenkinsParams -> + jenkinsParams.remove('MYHOSTTYPES_LIST') + jenkinsParams.remove('MYHOSTTYPES_TEST') + jenkinsParams.remove('MYHOSTTYPES_SERIAL_PARALLEL') + jenkinsParams.remove('CLEAN_ON_FAILURE') + return jenkinsParams + }, + clMatrixAxesFilter: { axis -> + !(params.DNS_TEST == 'both' && axis['DNS_FORCE_DISABLE'] == 'true' && axis['CLOUD_REGION'] == 'esxifree/dougalab') && /* DNS is not supported in dougalab */ + !(axis['IMAGE_TESTED'] != '_ubuntu2004image' && axis['CLOUD_REGION'] == 'esxifree/dougalab') && /* Only _ubuntu2004image is supported in dougalab */ + !(axis['REDEPLOY_SCHEME'] == '_scheme_addallnew_rmdisk_rollback') && /* _scheme_addallnew_rmdisk_rollback is not supported with myhostttpes set */ + !(axis['REDEPLOY_SCHEME'] == '_scheme_rmvm_keepdisk_rollback' && axis['CLOUD_REGION'].split('/')[0] == 'azure') && /* _scheme_rmvm_keepdisk_rollback not supported in Azure */ + !(axis['REDEPLOY_SCHEME'] == '_scheme_rmvm_rmdisk_only' && axis['SCALEUPDOWN'] == 'scaledown') && /* _scheme_rmvm_rmdisk_only only supports scaling up */ + !(axis['REDEPLOY_SCHEME'] == '_scheme_rmvm_keepdisk_rollback' && (axis['SCALEUPDOWN'] == 'scaledown' || axis['SCALEUPDOWN'] == 'scaleup')) /* _scheme_rmvm_keepdisk_rollback does not support scaling */ + }, + clTaskMap: { axisEnvVars -> + node { + withEnv(axisEnvVars) { + withCredentials([string(credentialsId: "VAULT_PASSWORD_${env.BUILDENV.toUpperCase()}", variable: 'VAULT_PASSWORD_BUILDENV')]) { + env.VAULT_PASSWORD_BUILDENV = VAULT_PASSWORD_BUILDENV + } + sh 'printenv | sort' + def stageBuild = new cStageBuild([result: 'SUCCESS']) + HashMap cluster_vars_override = [:] + + if (env.IMAGE_TESTED) { + cluster_vars_override += [image: "{{${env.IMAGE_TESTED}}}"] + stageBuild.userParams.put("cluster_vars_override", "\\\'" + groovy.json.JsonOutput.toJson(cluster_vars_override).replace("\"", "\\\"") + "\\\'") //NOTE: NO SPACES are allowed in this!! + } + + if (env.REDEPLOY_SCHEME) { + if (params.MYHOSTTYPES_LIST == '') { + currentBuild.result = 'FAILURE' + stageBuild.result = 'FAILURE' + unstable('Stage failed! Error was: ' + err) // OR: 'error "Stage failure"' or 'throw new org.jenkinsci.plugins.workflow.steps.FlowInterruptedException(hudson.model.Result.FAILURE)', but both of these fail all future stages, preventing us calling the clean. + } + + stageBuild.userParams.put("skip_release_version_check", "true") + stageBuild.userParams.put("release_version", "1_0_0") + stage_cvops('deploy', stageBuild, { + build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'deploy'), string(name: 'REDEPLOY_SCHEME', value: ''), string(name: 'CANARY', value: 'none'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: true), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.getUserParamsString())] + }) + + // Update the clustervars with new scaled cluster size + if (env.SCALEUPDOWN == 'scaleup') { + cluster_vars_override += ["${env.BUILDENV}": [hosttype_vars: [sys: [vms_by_az: [b: 1, c: 1]]]]] // AZ 'c' is not set normally + } else if (env.SCALEUPDOWN == 'scaledown') { + cluster_vars_override += ["${env.BUILDENV}": [hosttype_vars: [sys: [vms_by_az: [b: 0, c: 0]]]]] // AZ 'b' is set normally + } + + if (cluster_vars_override.size()) { + stageBuild.userParams.put("cluster_vars_override", "\\\'" + groovy.json.JsonOutput.toJson(cluster_vars_override).replace("\"", "\\\"") + "\\\'") //NOTE: NO SPACES are allowed in this!! + } + + // Run the split redeploy over all hosttypes + stageBuild.userParams.put("release_version", "2_0_0") + params.MYHOSTTYPES_LIST.split(',').each({ my_host_type -> + stage_cvops("redeploy canary=start ($my_host_type)", stageBuild, { + build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'redeploy'), string(name: 'REDEPLOY_SCHEME', value: (env.REDEPLOY_SCHEME ? env.REDEPLOY_SCHEME : '')), string(name: 'CANARY', value: 'start'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: false), string(name: 'MYHOSTTYPES', value: my_host_type), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.getUserParamsString())] + }) + + stage_cvops("redeploy canary=finish ($my_host_type)", stageBuild, { + build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'redeploy'), string(name: 'REDEPLOY_SCHEME', value: (env.REDEPLOY_SCHEME ? env.REDEPLOY_SCHEME : '')), string(name: 'CANARY', value: 'finish'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: false), string(name: 'MYHOSTTYPES', value: my_host_type), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.getUserParamsString())] + }) + + stage_cvops("redeploy canary=tidy ($my_host_type)", stageBuild, { + build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'redeploy'), string(name: 'REDEPLOY_SCHEME', value: (env.REDEPLOY_SCHEME ? env.REDEPLOY_SCHEME : '')), string(name: 'CANARY', value: 'tidy'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: false), string(name: 'MYHOSTTYPES', value: my_host_type), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.getUserParamsString())] + }) + }) + + //Need to redeploy original cluster (without scaling), to test that scaling works with next redeploy test (canary=none) + if (env.SCALEUPDOWN == 'scaleup' || env.SCALEUPDOWN == 'scaledown') { + cluster_vars_override.remove("${env.BUILDENV}") + stageBuild.userParams.put("cluster_vars_override", "\\\'" + groovy.json.JsonOutput.toJson(cluster_vars_override).replace("\"", "\\\"") + "\\\'") + stageBuild.userParams.put("release_version", "2_5_0") + stage_cvops('deploy clean original (unscaled) for next test', stageBuild, { + build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'deploy'), string(name: 'REDEPLOY_SCHEME', value: ''), string(name: 'CANARY', value: 'none'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: true), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.getUserParamsString() + " -e clean=_all_")] + }) + + //Re-add the scaleup/down cmdline for next redeploy test (canary=none) + if (env.SCALEUPDOWN == 'scaleup') { + cluster_vars_override += ["${env.BUILDENV}": [hosttype_vars: [sys: [vms_by_az: [b: 1, c: 1]]]]] // AZ 'c' is not set normally + } else if (env.SCALEUPDOWN == 'scaledown') { + cluster_vars_override += ["${env.BUILDENV}": [hosttype_vars: [sys: [vms_by_az: [b: 0, c: 0]]]]] // AZ 'b' is set normally + } + stageBuild.userParams.put("cluster_vars_override", "\\\'" + groovy.json.JsonOutput.toJson(cluster_vars_override).replace("\"", "\\\"") + "\\\'") + } + + // Run the canary=none redeploy over all hosttypes + stageBuild.userParams.put("release_version", "3_0_0") + params.MYHOSTTYPES_LIST.split(',').each({ my_host_type -> + stage_cvops("redeploy canary=none ($my_host_type) (tidy_on_success)", stageBuild, { + build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'redeploy'), string(name: 'REDEPLOY_SCHEME', value: (env.REDEPLOY_SCHEME ? env.REDEPLOY_SCHEME : '')), string(name: 'CANARY', value: 'none'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: true), string(name: 'MYHOSTTYPES', value: my_host_type), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.getUserParamsString())] + }) + }) + + if (stageBuild.result == 'SUCCESS' || params.CLEAN_ON_FAILURE == 'true') { + stage('clean') { + if (stageBuild.result != 'SUCCESS') { + echo "Stage failure: Running clean-up on cluster..." + } + catchError { + build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'clean'), string(name: 'REDEPLOY_SCHEME', value: ''), string(name: 'CANARY', value: 'none'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: true), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.getUserParamsString())] + } + } + } + } else { + stage_cvops('Redeploy not requested', stageBuild, { + echo "Redeploy testing not requested" + }) + } } - catchError { - build job: 'clusterverse/clusterverse-ops', parameters: [string(name: 'APP_NAME', value: "cvtest-${env.BUILD_NUMBER}-${env.BUILD_HASH}"), string(name: 'CLOUD_REGION', value: env.CLOUD_REGION), string(name: 'BUILDENV', value: env.BUILDENV), string(name: 'CLUSTER_ID', value: env.CLUSTER_ID), booleanParam(name: 'DNS_FORCE_DISABLE', value: env.DNS_FORCE_DISABLE), string(name: 'DEPLOY_TYPE', value: 'clean'), string(name: 'REDEPLOY_SCHEME', value: ''), string(name: 'CANARY', value: 'none'), booleanParam(name: 'CANARY_TIDY_ON_SUCCESS', value: true), string(name: 'MYHOSTTYPES', value: ''), string(name: 'CV_GIT_URL', value: scmVars ? scmVars.getUserRemoteConfigs()[0].getUrl() : DEFAULT_CLUSTERVERSE_URL), string(name: 'CV_GIT_BRANCH', value: CV_OPS_GIT_BRANCH), string(name: 'USER_CMDLINE_VARS', value: stageBuild.userParamsString())] - } - } } - } else { - stage_cvops('Redeploy not requested', stageBuild, { - echo "Redeploy testing not requested" - }) - } } - } - } ]) @@ -354,60 +430,57 @@ CVTEST_MYHOSTTYPES = new MatrixBuilder([ // A check stage - no actual work stage('Check Environment') { - node { - sh 'printenv | sort' - println(params.inspect()) - if (params.BUILDENV == '') { -// currentBuild.result = 'ABORTED' -// error "BUILDENV not defined" - unstable("BUILDENV not defined") - throw new org.jenkinsci.plugins.workflow.steps.FlowInterruptedException(hudson.model.Result.ABORTED) + node { + sh 'printenv | sort' + println(params.inspect()) + if (params.BUILDENV == '') { + error "BUILDENV not defined" + } } - } } // A map to be loaded with matrices (of stages) HashMap matrixBuilds = [:] -// A 'self-test' matrix. Only outputs debug. +//// A 'self-test' matrix. Only outputs debug. //matrixBuilds["SELFTEST1 Matrix builds"] = { -// stage("SELFTEST Matrix builds") { -// echo("Matrix 'params' used to build Matrix axes: \n" + SELFTEST._getMatrixParams().inspect() + "\n") -// echo("Matrix axes: \n" + SELFTEST._getMatrixAxes().inspect() + "\n") -// parallel(SELFTEST.getTaskMap()) -// } +// stage("SELFTEST Matrix builds") { +// echo("Matrix 'params' used to build Matrix axes: \n" + SELFTEST._getMatrixParams().inspect() + "\n") +// echo("Matrix axes: \n" + SELFTEST._getMatrixAxes().inspect() + "\n") +// parallel(SELFTEST.getTaskMap()) +// } //} // A matrix of tests that test pipelines *without* myhosttypes configured if (params.MYHOSTTYPES_TEST.split(',').contains('nomyhosttypes')) { - matrixBuilds["NOMYHOSTTYPES Matrix builds"] = { - stage("NOMYHOSTTYPES Matrix builds") { - echo("Matrix 'params' used to build Matrix axes: \n" + CVTEST_NOMYHOSTTYPES._getMatrixParams().inspect() + "\n") - echo("Matrix axes: \n" + CVTEST_NOMYHOSTTYPES._getMatrixAxes().inspect() + "\n") - parallel(CVTEST_NOMYHOSTTYPES.getTaskMap()) + matrixBuilds["NOMYHOSTTYPES Matrix builds"] = { + stage("NOMYHOSTTYPES Matrix builds") { + echo("Matrix 'params' used to build Matrix axes: \n" + CVTEST_NOMYHOSTTYPES._getMatrixParams().inspect() + "\n") + echo("Matrix axes: \n" + CVTEST_NOMYHOSTTYPES._getMatrixAxes().inspect() + "\n") + parallel(CVTEST_NOMYHOSTTYPES.getTaskMap()) + } } - } } // A matrix of tests that test pipelines *with* myhosttypes configured if (params.MYHOSTTYPES_TEST.split(',').contains('myhosttypes')) { - matrixBuilds["MYHOSTTYPES Matrix builds"] = { - stage("MYHOSTTYPES Matrix builds") { - echo("Matrix 'params' used to build Matrix axes: \n" + CVTEST_MYHOSTTYPES._getMatrixParams().inspect() + "\n") - echo("Matrix axes: \n" + CVTEST_MYHOSTTYPES._getMatrixAxes().inspect() + "\n") - parallel(CVTEST_MYHOSTTYPES.getTaskMap()) + matrixBuilds["MYHOSTTYPES Matrix builds"] = { + stage("MYHOSTTYPES Matrix builds") { + echo("Matrix 'params' used to build Matrix axes: \n" + CVTEST_MYHOSTTYPES._getMatrixParams().inspect() + "\n") + echo("Matrix axes: \n" + CVTEST_MYHOSTTYPES._getMatrixAxes().inspect() + "\n") + parallel(CVTEST_MYHOSTTYPES.getTaskMap()) + } } - } } // Run the matrices in parallel if the MYHOSTTYPES_SERIAL_PARALLEL parameter is set (makes in mess in Blue Ocean, but is faster). Else run serially. if (params.MYHOSTTYPES_SERIAL_PARALLEL == 'parallel') { - stage("All matrices") { - parallel(matrixBuilds) - } + stage("All matrices") { + parallel(matrixBuilds) + } } else { - matrixBuilds.each { matrix -> - matrix.value.call() - } + matrixBuilds.each { matrix -> + matrix.value.call() + } } diff --git a/readiness/tasks/remove_maintenance_mode_azure.yml b/readiness/tasks/remove_maintenance_mode_azure.yml new file mode 100644 index 00000000..e130f34a --- /dev/null +++ b/readiness/tasks/remove_maintenance_mode_azure.yml @@ -0,0 +1,30 @@ +--- + +- name: remove_maintenance_mode/azure | Set maintenance_mode=false asynchronously + azure.azcollection.azure_rm_virtualmachine: + client_id: "{{cluster_vars[buildenv].azure_client_id}}" + secret: "{{cluster_vars[buildenv].azure_secret}}" + subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" + tenant: "{{cluster_vars[buildenv].azure_tenant}}" + resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" + append_tags: yes + tags: + maintenance_mode: "false" + name: "{{ item.name }}" + zones: ["{{ (item.regionzone.split('-'))[1] }}"] + register: r__azure_rm_virtualmachine + with_items: "{{ cluster_hosts_state }}" + delegate_to: localhost + run_once: true + async: 7200 + poll: 0 + +- name: remove_maintenance_mode/azure | Wait for maintenance_mode labelling to finish + async_status: + jid: "{{ item.ansible_job_id }}" + register: async_jobs + until: async_jobs.finished + retries: 300 + with_items: "{{r__azure_rm_virtualmachine.results}}" + delegate_to: localhost + run_once: true diff --git a/readiness/tasks/remove_maintenance_mode_esxifree.yml b/readiness/tasks/remove_maintenance_mode_esxifree.yml new file mode 100644 index 00000000..20a4d0e7 --- /dev/null +++ b/readiness/tasks/remove_maintenance_mode_esxifree.yml @@ -0,0 +1,11 @@ +--- + +- name: remove_maintenance_mode/esxifree | Set maintenance_mode to false + esxifree_guest: + hostname: "{{ cluster_vars.esxi_ip }}" + username: "{{ cluster_vars.username }}" + password: "{{ cluster_vars.password }}" + name: "{{item.name}}" + state: "unchanged" + annotation: "{{ item.tagslabels | combine({'maintenance_mode': 'false'}) }}" + with_items: "{{ cluster_hosts_state }}" diff --git a/redeploy/__common/tasks/powerchange_vms_azure.yml b/redeploy/__common/tasks/powerchange_vms_azure.yml new file mode 100644 index 00000000..c49996e9 --- /dev/null +++ b/redeploy/__common/tasks/powerchange_vms_azure.yml @@ -0,0 +1,32 @@ +--- + +- name: "powerchange_vms/azure | hosts_to_powerchange (to {{powerchange_new_state}})" + debug: msg="{{hosts_to_powerchange}}" + +- name: "powerchange_vms/azure | {{powerchange_new_state}} VM(s) and set maintenance_mode=true (if stopping)" + block: + - name: "powerchange_vms/azure | {{powerchange_new_state}} VMs asynchronously and set maintenance_mode=true (if stopping)" + azure.azcollection.azure_rm_virtualmachine: + client_id: "{{cluster_vars[buildenv].azure_client_id}}" + secret: "{{cluster_vars[buildenv].azure_secret}}" + subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" + tenant: "{{cluster_vars[buildenv].azure_tenant}}" + resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" + append_tags: yes + name: "{{ item.name }}" + tags: "{% if powerchange_new_state == 'stop' %}{'maintenance_mode': 'true'}{% else %}{{omit}}{% endif %}" + started: "{% if powerchange_new_state == 'stop' %}no{% else %}yes{% endif %}" + zones: ["{{ (item.regionzone.split('-'))[1] }}"] + register: r__azure_rm_virtualmachine + with_items: "{{ hosts_to_powerchange }}" + async: 7200 + poll: 0 + + - name: "powerchange_vms/azure | Wait for VM(s) to {{powerchange_new_state}}" + async_status: + jid: "{{ item.ansible_job_id }}" + register: async_jobs + until: async_jobs.finished + retries: 300 + with_items: "{{r__azure_rm_virtualmachine.results}}" + when: hosts_to_powerchange | length diff --git a/redeploy/__common/tasks/powerchange_vms_esxifree.yml b/redeploy/__common/tasks/powerchange_vms_esxifree.yml new file mode 100644 index 00000000..c62d3593 --- /dev/null +++ b/redeploy/__common/tasks/powerchange_vms_esxifree.yml @@ -0,0 +1,28 @@ +--- + +- name: "powerchange_vms/esxifree | hosts_to_powerchange (to {{powerchange_new_state}})" + debug: msg="{{hosts_to_powerchange}}" + +- name: "powerchange_vms/esxifree | {{powerchange_new_state}} VM(s) and set maintenance_mode=true" + block: + - name: powerchange_vms/esxifree | Set maintenance_mode=true (if stopping) + esxifree_guest: + hostname: "{{ cluster_vars.esxi_ip }}" + username: "{{ cluster_vars.username }}" + password: "{{ cluster_vars.password }}" + name: "{{item.name}}" + state: unchanged + annotation: "{{ item.tagslabels | combine({'maintenance_mode': 'true'}) }}" + with_items: "{{ hosts_to_powerchange }}" + when: "powerchange_new_state == 'stop'" + + - name: "powerchange_vms/esxifree | {{powerchange_new_state}} VMs asynchronously" + esxifree_guest: + hostname: "{{ cluster_vars.esxi_ip }}" + username: "{{ cluster_vars.username }}" + password: "{{ cluster_vars.password }}" + name: "{{item.name}}" + state: "{% if powerchange_new_state == 'stop' %}shutdownguest{% else %}poweredon{% endif %}" + with_items: "{{ hosts_to_powerchange }}" + when: hosts_to_powerchange | length + \ No newline at end of file diff --git a/redeploy/__common/tasks/set_lifecycle_state_label_azure.yml b/redeploy/__common/tasks/set_lifecycle_state_label_azure.yml new file mode 100644 index 00000000..6aa7060d --- /dev/null +++ b/redeploy/__common/tasks/set_lifecycle_state_label_azure.yml @@ -0,0 +1,18 @@ +--- + +- name: set_lifecycle_state_label/azure | hosts_to_relabel + debug: msg="{{hosts_to_relabel}}" + +- name: "set_lifecycle_state_label/azure | Change lifecycle_state label to {{new_state}}" + azure.azcollection.azure_rm_virtualmachine: + client_id: "{{cluster_vars[buildenv].azure_client_id}}" + secret: "{{cluster_vars[buildenv].azure_secret}}" + subscription_id: "{{cluster_vars[buildenv].azure_subscription_id}}" + tenant: "{{cluster_vars[buildenv].azure_tenant}}" + resource_group: "{{cluster_vars[buildenv].azure_resource_group}}" + append_tags: yes + name: "{{ item.name }}" + tags: + lifecycle_state: "{{ new_state }}" + zones: ["{{ (item.regionzone.split('-'))[1] }}"] + with_items: "{{ hosts_to_relabel | default([]) }}" diff --git a/redeploy/__common/tasks/set_lifecycle_state_label_esxifree.yml b/redeploy/__common/tasks/set_lifecycle_state_label_esxifree.yml new file mode 100644 index 00000000..8598d6ec --- /dev/null +++ b/redeploy/__common/tasks/set_lifecycle_state_label_esxifree.yml @@ -0,0 +1,14 @@ +--- + +- name: set_lifecycle_state_label/esxifree | hosts_to_relabel + debug: msg="{{hosts_to_relabel}}" + +- name: "set_lifecycle_state_label/esxifree | Change lifecycle_state label to {{new_state}}" + esxifree_guest: + hostname: "{{ cluster_vars.esxi_ip }}" + username: "{{ cluster_vars.username }}" + password: "{{ cluster_vars.password }}" + name: "{{item.name}}" + state: "unchanged" + annotation: "{{ item.tagslabels | combine({'lifecycle_state': new_state}) }}" + with_items: "{{ hosts_to_relabel | default([]) }}" diff --git a/redeploy/_scheme_rmvm_keepdisk_rollback/tasks/_add_src_diskinfo_to_cluster_hosts_target__esxifree.yml b/redeploy/_scheme_rmvm_keepdisk_rollback/tasks/_add_src_diskinfo_to_cluster_hosts_target__esxifree.yml new file mode 100644 index 00000000..85a07832 --- /dev/null +++ b/redeploy/_scheme_rmvm_keepdisk_rollback/tasks/_add_src_diskinfo_to_cluster_hosts_target__esxifree.yml @@ -0,0 +1,26 @@ +--- + +- name: _add_src_diskinfo_to_cluster_hosts_target/esxifree | cluster_hosts_state + debug: msg={{cluster_hosts_state}} + +- assert: { that: "cluster_hosts_state | json_query(\"[].disk_info_cloud.*[] | [?backing_datastore!='\" + cluster_vars.datastore + \"']\") | length == 0", msg: "Move is only possible if disks are on the same datastore." } + when: _scheme_rmvm_keepdisk_rollback__copy_or_move == "move" + +- name: _add_src_diskinfo_to_cluster_hosts_target/esxifree | augment cluster_hosts_target auto_volumes with source disk info + set_fact: + cluster_hosts_target: | + {%- for cht_host in cluster_hosts_target -%} + {%- for cht_autovol in cht_host.auto_volumes -%} + {%- for chs_vm in cluster_hosts_state | selectattr('tagslabels.lifecycle_state', '!=', 'current')-%} + {%- if cht_host.hostname | regex_replace('-(?!.*-).*') == chs_vm.name | regex_replace('-(?!.*-).*') -%} + {%- for chs_host_diskinfo in chs_vm.disk_info_cloud | to_json | from_json | json_query('[?unit_number!=`0` && backing_type==\'FlatVer2\' && contains(backing_filename, \'--' + cht_autovol.volname + '.vmdk\')]') -%} + {%- set _ = cht_autovol.update({'volume_size': (chs_host_diskinfo.capacity_in_bytes/1073741824)|int, 'src': {'backing_filename': chs_host_diskinfo.backing_filename, 'copy_or_move': _scheme_rmvm_keepdisk_rollback__copy_or_move }}) -%} + {%- endfor -%} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + {%- endfor -%} + {{cluster_hosts_target}} + +- name: _add_src_diskinfo_to_cluster_hosts_target/esxifree | cluster_hosts_target + debug: msg={{cluster_hosts_target}}