diff --git a/cli/engine/ApplyEngine.py b/cli/engine/ApplyEngine.py index 836b0e4265..7dbbe4c090 100644 --- a/cli/engine/ApplyEngine.py +++ b/cli/engine/ApplyEngine.py @@ -1,13 +1,10 @@ import os import sys -from ansible.parsing.dataloader import DataLoader -from ansible.inventory.manager import InventoryManager - from cli.version import VERSION from cli.helpers.Step import Step from cli.helpers.doc_list_helpers import select_single, select_all, select_first -from cli.helpers.build_io import save_manifest, load_manifest, get_inventory_path, get_manifest_path, get_build_path +from cli.helpers.build_io import save_manifest, load_manifest, get_inventory_path, get_manifest_path, get_build_path, load_inventory from cli.helpers.yaml_helpers import safe_load_all from cli.helpers.Log import Log from cli.helpers.os_images import get_os_distro_normalized @@ -39,7 +36,7 @@ def __init__(self, input_data): self.infrastructure_docs = [] self.manifest_docs = [] - self.__ping_retries: int = input_data.ping_retries + self.ping_retries: int = input_data.ping_retries def __enter__(self): return self @@ -121,7 +118,7 @@ def assert_no_master_downscale(self): inventory_path = get_inventory_path(cluster_name) if os.path.isfile(inventory_path): - existing_inventory = InventoryManager(loader=DataLoader(), sources=inventory_path) + existing_inventory = load_inventory(inventory_path) both_present = all([ 'kubernetes_master' in existing_inventory.list_groups(), @@ -149,7 +146,7 @@ def assert_no_postgres_nodes_number_change(self): if os.path.isfile(inventory_path): next_postgres_node_count = 0 - existing_inventory = InventoryManager(loader=DataLoader(), sources=inventory_path) + existing_inventory = load_inventory(inventory_path) prev_postgres_node_count = len(existing_inventory.list_hosts(pattern='postgresql')) postgres_available = [x for x in feature_mapping.specification.available_roles if x.name == 'postgresql'] if postgres_available[0].enabled: @@ -196,7 +193,7 @@ def apply(self): self.assert_consistent_os_family() - if not (self.skip_infrastructure or self.is_provider_any(self.cluster_model)): + if not (self.skip_infrastructure or self.cluster_model['provider'] == 'any'): # Generate terraform templates with TerraformTemplateGenerator(self.cluster_model, self.infrastructure_docs) as template_generator: template_generator.run() @@ -222,19 +219,7 @@ def apply(self): # Run Ansible to provision infrastructure if not(self.skip_config): with AnsibleRunner(self.cluster_model, docs, ansible_options=self.ansible_options, - ping_retries=self.__ping_retries) as ansible_runner: + ping_retries=self.ping_retries) as ansible_runner: ansible_runner.apply() return 0 - - def dry_run(self): - - self.process_input_docs() - - self.process_configuration_docs() - - return [*self.configuration_docs, *self.infrastructure_docs] - - @staticmethod - def is_provider_any(cluster_model): - return cluster_model["provider"] == "any" diff --git a/cli/engine/InitEngine.py b/cli/engine/InitEngine.py index 3f021f78a9..3a9efe62c4 100644 --- a/cli/engine/InitEngine.py +++ b/cli/engine/InitEngine.py @@ -3,10 +3,11 @@ from cli.helpers.Step import Step from cli.helpers.build_io import save_manifest, get_build_path from cli.helpers.data_loader import load_all_schema_objs, types -from cli.engine.ApplyEngine import ApplyEngine from cli.helpers.objdict_helpers import remove_value from cli.version import VERSION -from cli.helpers.doc_list_helpers import select_all, select_single +from cli.helpers.doc_list_helpers import select_all +from cli.engine.schema.ConfigurationAppender import ConfigurationAppender +from cli.engine.schema.DefaultMerger import DefaultMerger class InitEngine(Step): @@ -25,23 +26,29 @@ def __exit__(self, exc_type, exc_value, traceback): pass def init(self): - input = load_all_schema_objs(types.DEFAULT, self.provider, 'configuration/minimal-cluster-config') - input[0].specification.name = self.name + # Load the minimal cluster-config doc and set the cluster name + docs = load_all_schema_objs(types.DEFAULT, self.provider, 'configuration/minimal-cluster-config') + docs[0].specification.name = self.name + # For full we also add the infrastructure and configuration documents if self.is_full_config: - config = self.get_config_docs(input) - config_only = select_all(config, lambda x: not(x.kind.startswith('epiphany-cluster'))) - if self.provider == 'any': - # for any provider we want to use the default config from minimal-cluster-config - cluster_model = select_single(input, lambda x: x.kind == 'epiphany-cluster') + # Merge with defaults + with DefaultMerger(docs) as doc_merger: + docs = doc_merger.run() + + # Add infrastructure and configuration documents + if self.provider != 'any': + # Add VM infrastructure docs as these are most likely to be changed + infra_docs = load_all_schema_objs(types.DEFAULT, self.provider, 'infrastructure/virtual-machine') else: - # for azure|aws provider we want to use the extended defaults cluster-config after dry run. - # TODO: We probably wants this comming from seperate documents since Azure and AWS overlap now... - cluster_model = select_single(config, lambda x: x.kind == 'epiphany-cluster') - infra = self.get_infra_docs(input) - docs = [cluster_model, *config_only, *infra] - else: - docs = [*input] + # For any provider, infrastructure docs are already part of the minimal-cluster-config template + infra_docs = select_all(docs, lambda x: x.kind.startswith('infrastructure/machine')) + + # Add configuration documents + with ConfigurationAppender(docs) as config_appender: + config_docs = config_appender.run() + + docs = [*config_docs, *infra_docs] # set the provider and version for all docs for doc in docs: @@ -52,35 +59,7 @@ def init(self): remove_value(docs, 'SET_BY_AUTOMATION') # save document - save_manifest(docs, self.name, self.name+'.yml') + save_manifest(docs, self.name, f'{ self.name }.yml') self.logger.info('Initialized new configuration and saved it to "' + os.path.join(get_build_path(self.name), self.name + '.yml') + '"') return 0 - - def get_config_docs(self, input_docs): - cluster_config_path = save_manifest(input_docs, self.name, self.name + '.yml') - args = type('obj', (object,), {'file': cluster_config_path})() - args.ansible_forks = 10 # TODO: remove these workarounds - args.ping_retries = 5 - - # generate the config documents - with ApplyEngine(args) as build: - config = build.dry_run() - - return config - - def get_infra_docs(self, input_docs): - if self.provider == 'any': - # For any we can include the machine documents from the minimal-cluster-config - infra = select_all(input_docs, lambda x: x.kind.startswith('infrastructure/machine')) - else: - # VMs are curently the infrastructure documents the user might interact with for: - # - type/size - # - distro - # - network security rules - # ... - # So we add the defaults here. - # TODO: Check if we want to include possible other infrastructure documents. - infra = load_all_schema_objs(types.DEFAULT, self.provider, 'infrastructure/virtual-machine') - - return infra diff --git a/cli/engine/UpgradeEngine.py b/cli/engine/UpgradeEngine.py index 0fc7bad0df..5754eb6195 100644 --- a/cli/engine/UpgradeEngine.py +++ b/cli/engine/UpgradeEngine.py @@ -21,7 +21,7 @@ def __init__(self, input_data): self.backup_build_dir = '' self.ansible_command = AnsibleCommand() self.input_docs = [] - self.__ping_retries: int = input_data.ping_retries + self.ping_retries: int = input_data.ping_retries def __enter__(self): super().__enter__() @@ -88,7 +88,7 @@ def upgrade(self): # Run Ansible to upgrade infrastructure with AnsibleRunner(build_dir=self.build_dir, backup_build_dir=self.backup_build_dir, ansible_options=self.ansible_options, config_docs=self.input_docs, - ping_retries=self.__ping_retries) as ansible_runner: + ping_retries=self.ping_retries) as ansible_runner: ansible_runner.upgrade() return 0 diff --git a/cli/engine/ansible/AnsibleInventoryUpgrade.py b/cli/engine/ansible/AnsibleInventoryUpgrade.py index 67ddc5a515..61e8a3085c 100644 --- a/cli/engine/ansible/AnsibleInventoryUpgrade.py +++ b/cli/engine/ansible/AnsibleInventoryUpgrade.py @@ -1,12 +1,8 @@ -from ansible.parsing.dataloader import DataLoader -from ansible.inventory.manager import InventoryManager - from cli.helpers.Step import Step from cli.helpers.build_io import get_inventory_path_for_build from cli.models.AnsibleHostModel import AnsibleHostModel from cli.models.AnsibleInventoryItem import AnsibleInventoryItem -from cli.helpers.build_io import save_inventory, load_manifest -from cli.helpers.objdict_helpers import dict_to_objdict +from cli.helpers.build_io import save_inventory, load_manifest, load_inventory from cli.helpers.data_loader import load_schema_obj, types as data_types from cli.helpers.doc_list_helpers import select_single from cli.helpers.objdict_helpers import merge_objdict @@ -56,7 +52,7 @@ def upgrade(self): inventory_path = get_inventory_path_for_build(self.backup_build_dir) self.logger.info(f'Loading backup Ansible inventory: {inventory_path}') - loaded_inventory = InventoryManager(loader = DataLoader(), sources=inventory_path) + loaded_inventory = load_inventory(inventory_path) # move loaded inventory to templating structure new_inventory = [] diff --git a/cli/engine/ansible/AnsibleRunner.py b/cli/engine/ansible/AnsibleRunner.py index eaff61ca7d..58ada62606 100644 --- a/cli/engine/ansible/AnsibleRunner.py +++ b/cli/engine/ansible/AnsibleRunner.py @@ -27,7 +27,7 @@ def __init__(self, cluster_model=None, config_docs=None, build_dir=None, backup_ self.backup_build_dir = backup_build_dir self.ansible_options = ansible_options self.ansible_command = AnsibleCommand() - self.__ping_retries: int = ping_retries + self.ping_retries: int = ping_retries def __enter__(self): super().__enter__() @@ -57,7 +57,7 @@ def pre_flight(self, inventory_path): self.ansible_command.run_task_with_retries(inventory=inventory_path, module="ping", hosts="all", - retries=self.__ping_retries) + retries=self.ping_retries) self.logger.info('Checking preflight conditions on each machine') self.ansible_command.run_playbook_with_retries(inventory=inventory_path, @@ -106,7 +106,6 @@ def apply(self): # create inventory inventory_creator = AnsibleInventoryCreator(self.cluster_model, self.config_docs) inventory_creator.create() - time.sleep(10) # create ansible.cfg ansible_config_file_path = get_ansible_config_file_path(self.cluster_model.specification.name) diff --git a/cli/engine/schema/ConfigurationAppender.py b/cli/engine/schema/ConfigurationAppender.py index 3f0f92dc74..d8b76ec554 100644 --- a/cli/engine/schema/ConfigurationAppender.py +++ b/cli/engine/schema/ConfigurationAppender.py @@ -7,7 +7,7 @@ class ConfigurationAppender(Step): - REQUIRED_DOCS = ['configuration/feature-mapping', 'configuration/shared-config', 'epiphany-cluster'] + REQUIRED_DOCS = ['epiphany-cluster', 'configuration/feature-mapping', 'configuration/shared-config'] def __init__(self, input_docs): super().__init__(__name__) diff --git a/cli/epicli.py b/cli/epicli.py index aaca209adf..c32610ae1a 100644 --- a/cli/epicli.py +++ b/cli/epicli.py @@ -490,9 +490,7 @@ def dump_external_debug_info(title, args): dump_file.write('\n\n*****PYTHON******\n') dump_file.write(f'python_version: {platform.python_version()}\n') dump_file.write(f'python_build: {platform.python_build()}\n') - dump_file.write(f'python_revision: {platform.python_revision()}\n') dump_file.write(f'python_compiler: {platform.python_compiler()}\n') - dump_file.write(f'python_branch: {platform.python_branch()}\n') dump_file.write(f'python_implementation: {platform.python_implementation()}\n') dump_external_debug_info('ANSIBLE VERSION', ['ansible', '--version']) @@ -501,7 +499,7 @@ def dump_external_debug_info(title, args): dump_external_debug_info('TERRAFORM VERSION', ['terraform', '--version']) dump_external_debug_info('RUBY VERSION', ['ruby', '--version']) dump_external_debug_info('RUBY GEM VERSION', ['gem', '--version']) - dump_external_debug_info('RUBY INSTALLED GEMS', ['gem', 'query', '--local']) + dump_external_debug_info('RUBY INSTALLED GEMS', ['gem', 'list', '--local']) dump_file.write('\n\n*****LOG******\n') log_path = os.path.join(get_output_path(), config.log_file) diff --git a/cli/helpers/build_io.py b/cli/helpers/build_io.py index 0001851980..6f6efa076e 100644 --- a/cli/helpers/build_io.py +++ b/cli/helpers/build_io.py @@ -5,6 +5,9 @@ from os.path import isfile, join from distutils import dir_util +from ansible.parsing.dataloader import DataLoader +from ansible.inventory.manager import InventoryManager + from cli.helpers.data_loader import load_template_file, load_yamls_file, types from cli.helpers.yaml_helpers import dump_all, dump from cli.helpers.Config import Config @@ -53,6 +56,10 @@ def save_inventory(inventory, cluster_model, build_dir=None): save_to_file(file_path, content) +def load_inventory(inventory_path): + return InventoryManager(loader=DataLoader(), sources=inventory_path) + + def save_ansible_config_file(ansible_config_file_settings, ansible_config_file_path): template = load_template_file(types.ANSIBLE, '', ANSIBLE_CFG_FILE) content = template.render(ansible_config_file_settings=ansible_config_file_settings) diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md index d886942d14..4cc4dfd66c 100644 --- a/docs/changelogs/CHANGELOG-2.0.md +++ b/docs/changelogs/CHANGELOG-2.0.md @@ -14,8 +14,9 @@ - [#1569](https://github.com/epiphany-platform/epiphany/issues/1569) - Azure unmanaged disks not supported by Epiphany but there is misleading setting in the default configuration - [#2832](https://github.com/epiphany-platform/epiphany/issues/2832) - Make the DoD checklist clear - [#2853](https://github.com/epiphany-platform/epiphany/issues/2853) - Change autoscaling_group approach in AWS provider in favor of plain VM creation. - - [#2669](https://github.com/epiphany-platform/epiphany/issues/2669) - Restarting the installation process can cause certificate problems if K8s was not fully configured +- [#2944](https://github.com/epiphany-platform/epiphany/issues/2944) - Refactor InitEngine class to be agnostic to changes in ApplyEngine and UpgradeEngine +- [#2945](https://github.com/epiphany-platform/epiphany/issues/2945) - epicli apply sleeps 10 seconds after creating inventory ### Updated diff --git a/schema/any/defaults/epiphany-cluster.yml b/schema/any/defaults/epiphany-cluster.yml index 4bc8719823..27a3014ac2 100644 --- a/schema/any/defaults/epiphany-cluster.yml +++ b/schema/any/defaults/epiphany-cluster.yml @@ -11,45 +11,45 @@ specification: components: kubernetes_master: count: 1 - machine: kubernetes-master-machine + machines: [] configuration: default kubernetes_node: count: 2 - machine: kubernetes-node-machine + machines: [] configuration: default logging: count: 1 - machine: logging-machine + machines: [] configuration: default monitoring: count: 1 - machine: monitoring-machine + machines: [] configuration: default kafka: count: 2 - machine: kafka-machine + machines: [] configuration: default postgresql: count: 0 - machine: postgresql-machine + machines: [] configuration: default load_balancer: count: 1 - machine: load-balancer-machine + machines: [] configuration: default rabbitmq: count: 0 - machine: rabbitmq-machine + machines: [] configuration: default opendistro_for_elasticsearch: count: 0 - machine: logging-machine + machines: [] configuration: default repository: count: 1 - machine: repository-machine + machines: [] configuration: default single_machine: count: 0 - machine: single-machine + machines: [] configuration: default