From e5957c55d2e78c51fcddd461043d0347af256d2b Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Mon, 13 Jul 2020 17:04:14 +0200 Subject: [PATCH] epicli/upgrade: reusing existing shared-config + cleanups --- core/src/epicli/cli/engine/UpgradeEngine.py | 10 ++--- .../engine/ansible/AnsibleInventoryUpgrade.py | 39 +++++++++++++------ .../engine/ansible/AnsibleVarsGenerator.py | 6 +-- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/core/src/epicli/cli/engine/UpgradeEngine.py b/core/src/epicli/cli/engine/UpgradeEngine.py index 756d20a119..9e15469d58 100644 --- a/core/src/epicli/cli/engine/UpgradeEngine.py +++ b/core/src/epicli/cli/engine/UpgradeEngine.py @@ -28,24 +28,24 @@ def get_backup_dirs(self): for d in os.listdir(self.build_dir): bd = os.path.join(self.build_dir, d) if os.path.isdir(bd) and re.match(r'backup_\d', d): result.append(bd) - return result + return result def backup_build(self): - # check if there are backup dirs and if so take the latest to work with. + # Check if there are backup dirs and if so take the latest to work with backup_dirs = self.get_backup_dirs() if len(backup_dirs) > 0: self.backup_build_dir = max(backup_dirs , key=os.path.getmtime) self.logger.info(f'There is already a backup present. Using latest for upgrade: "{self.backup_build_dir}"') return - # no backup dir so use the latest + # No backup dir so use the latest backup_dir_name = f'backup_{int(round(time.time() * 1000))}' self.backup_build_dir = os.path.join(self.build_dir, backup_dir_name ) self.logger.info(f'Backing up build dir to "{self.backup_build_dir}"') shutil.copytree(self.build_dir, self.backup_build_dir) def upgrade(self): - # backup existing build + # Backup existing build self.backup_build() # Run Ansible to upgrade infrastructure @@ -53,4 +53,4 @@ def upgrade(self): ansible_options=self.ansible_options) as ansible_runner: ansible_runner.upgrade() - return 0 \ No newline at end of file + return 0 diff --git a/core/src/epicli/cli/engine/ansible/AnsibleInventoryUpgrade.py b/core/src/epicli/cli/engine/ansible/AnsibleInventoryUpgrade.py index d0a85450d4..70e8f06e7c 100644 --- a/core/src/epicli/cli/engine/ansible/AnsibleInventoryUpgrade.py +++ b/core/src/epicli/cli/engine/ansible/AnsibleInventoryUpgrade.py @@ -1,11 +1,17 @@ +import os + from ansible.parsing.dataloader import DataLoader from ansible.inventory.manager import InventoryManager + from cli.helpers.Step import Step from cli.helpers.build_saver import get_inventory_path_for_build, check_build_output_version, BUILD_LEGACY +from cli.helpers.build_saver import save_inventory, MANIFEST_FILE_NAME +from cli.helpers.data_loader import load_yamls_file +from cli.helpers.objdict_helpers import dict_to_objdict +from cli.helpers.doc_list_helpers import select_single + from cli.models.AnsibleHostModel import AnsibleHostModel from cli.models.AnsibleInventoryItem import AnsibleInventoryItem -from cli.helpers.build_saver import save_inventory -from cli.helpers.objdict_helpers import dict_to_objdict class AnsibleInventoryUpgrade(Step): @@ -14,13 +20,14 @@ def __init__(self, build_dir, backup_build_dir): self.build_dir = build_dir self.backup_build_dir = backup_build_dir self.cluster_model = None + self.shared_config = None def __enter__(self): super().__enter__() return self def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) + super().__exit__(exc_type, exc_value, traceback) def get_role(self, inventory, role_name): for role in inventory: @@ -32,7 +39,7 @@ def delete_role(self, inventory, role_name): for i in range(len(inventory)): if inventory[i].role == role_name: del inventory[i] - return + return def rename_role(self, inventory, role_name, new_role_name): role = self.get_role(inventory, role_name) @@ -40,13 +47,13 @@ def rename_role(self, inventory, role_name, new_role_name): role.role = new_role_name def upgrade(self): - inventory_path = get_inventory_path_for_build(self.backup_build_dir) + inventory_path = get_inventory_path_for_build(self.backup_build_dir) build_version = check_build_output_version(self.backup_build_dir) self.logger.info(f'Loading backup Ansible inventory: {inventory_path}') loaded_inventory = InventoryManager(loader = DataLoader(), sources=inventory_path) - # move loaded inventory to templating structure + # Move loaded inventory to templating structure new_inventory = [] for key in loaded_inventory.groups: if key != 'all' and key != 'ungrouped': @@ -56,7 +63,7 @@ def upgrade(self): new_hosts.append(AnsibleHostModel(host.address, host.vars['ansible_host'])) new_inventory.append(AnsibleInventoryItem(key, new_hosts)) - # re-constructure cluster model with all data necessary to run required upgrade rolls + # Reconstruct cluster model with all data necessary to run required upgrade rolls self.cluster_model = dict_to_objdict({ 'provider': 'any', 'specification': { @@ -67,6 +74,14 @@ def upgrade(self): } }) + # Reuse shared config from existing manifest + # Shared config contains the use_ha_control_plane flag which is required during upgrades + path_to_manifest = os.path.join(self.backup_build_dir, MANIFEST_FILE_NAME) + if not os.path.isfile(path_to_manifest): + raise Exception('No manifest.yml inside the build folder') + manifest_docs = load_yamls_file(path_to_manifest) + self.shared_config = select_single(manifest_docs, lambda x: x.kind == 'configuration/shared-config') + if build_version == BUILD_LEGACY: self.logger.info(f'Upgrading Ansible inventory Epiphany < 0.3.0') @@ -79,7 +94,7 @@ def upgrade(self): self.rename_role(new_inventory, 'kafka-exporter', 'kafka_exporter') self.rename_role(new_inventory, 'haproxy_tls_termination', 'haproxy') - # remove linux and reboot roles if present + # Remove linux and reboot roles if present self.delete_role(new_inventory, 'linux') self.delete_role(new_inventory, 'reboot') else: @@ -91,21 +106,21 @@ def upgrade(self): raise Exception('No kubernetes_master to use as repository') master_node = master.hosts[0] - # add image_registry + # Add image_registry image_registry = self.get_role(new_inventory, 'image_registry') if image_registry == None: hosts = [] hosts.append(AnsibleHostModel(master_node.name, master_node.ip)) new_inventory.append(AnsibleInventoryItem('image_registry', hosts)) - # add repository + # Add repository repository = self.get_role(new_inventory, 'repository') if repository == None: hosts = [] hosts.append(AnsibleHostModel(master_node.name, master_node.ip)) new_inventory.append(AnsibleInventoryItem('repository', hosts)) - # save new inventory + # Save new inventory save_inventory(new_inventory, self.cluster_model, self.build_dir) - return 0 \ No newline at end of file + return 0 diff --git a/core/src/epicli/cli/engine/ansible/AnsibleVarsGenerator.py b/core/src/epicli/cli/engine/ansible/AnsibleVarsGenerator.py index f4853e5db1..016462b7c5 100644 --- a/core/src/epicli/cli/engine/ansible/AnsibleVarsGenerator.py +++ b/core/src/epicli/cli/engine/ansible/AnsibleVarsGenerator.py @@ -93,10 +93,10 @@ def populate_group_vars(self, ansible_dir): main_vars['is_upgrade_run'] = self.is_upgrade_run main_vars['roles_with_generated_vars'] = sorted(self.roles_with_generated_vars) - shared_config_doc = select_first(self.config_docs, lambda x: x.kind == 'configuration/shared-config') + shared_config_doc = self.inventory_upgrade.shared_config if shared_config_doc == None: shared_config_doc = load_yaml_obj(types.DEFAULT, 'common', 'configuration/shared-config') - + self.set_vault_path(shared_config_doc) main_vars.update(shared_config_doc.specification) @@ -115,7 +115,7 @@ def set_vault_path(self, shared_config): shared_config.specification.vault_tmp_file_location = Config().vault_password_location cluster_name = self.get_cluster_name() shared_config.specification.vault_location = get_ansible_vault_path(cluster_name) - + def get_cluster_name(self): if 'name' in self.cluster_model.specification.keys(): return self.cluster_model.specification.name