diff --git a/CHANGELOG-0.7.md b/CHANGELOG-0.7.md new file mode 100644 index 0000000000..796c88c60f --- /dev/null +++ b/CHANGELOG-0.7.md @@ -0,0 +1,9 @@ +# Changelog 0.7 + +## [0.7.0] 2020-0X-XX + +### Added + +#### General + +- [#811](https://github.com/epiphany-platform/epiphany/issues/811) - Measure execution time of Ansible tasks diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e32cbdb2d..6900ae98e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. Reference for actual cluster component versions can be found [here](docs/home/COMPONENTS.md) +- [CHANGELOG-0.7.x](./CHANGELOG-0.7.md) - [CHANGELOG-0.6.x](./CHANGELOG-0.6.md) - [CHANGELOG-0.5.x](./CHANGELOG-0.5.md) - [CHANGELOG-0.4.x](./CHANGELOG-0.4.md) diff --git a/core/src/epicli/cli/engine/ApplyEngine.py b/core/src/epicli/cli/engine/ApplyEngine.py index 877f21aee6..052d337129 100644 --- a/core/src/epicli/cli/engine/ApplyEngine.py +++ b/core/src/epicli/cli/engine/ApplyEngine.py @@ -20,8 +20,9 @@ class ApplyEngine(Step): def __init__(self, input_data): self.file = input_data.file - self.skip_infrastructure = input_data.no_infra if hasattr(input_data, 'no_infra') else False - self.skip_config = input_data.skip_config if hasattr(input_data, 'skip_config') else False + self.skip_infrastructure = getattr(input_data, 'no_infra', False) + self.skip_config = getattr(input_data, 'skip_config', False) + self.ansible_options = {'profile_tasks': getattr(input_data, 'profile_ansible_tasks', False)} self.logger = Log(__name__) self.cluster_model = None @@ -147,7 +148,7 @@ def apply(self): # Run Ansible to provision infrastructure if not(self.skip_config): - with AnsibleRunner(self.cluster_model, docs) as ansible_runner: + with AnsibleRunner(self.cluster_model, docs, ansible_options=self.ansible_options) as ansible_runner: ansible_runner.apply() return 0 diff --git a/core/src/epicli/cli/engine/UpgradeEngine.py b/core/src/epicli/cli/engine/UpgradeEngine.py index d053c66037..ac40496f02 100644 --- a/core/src/epicli/cli/engine/UpgradeEngine.py +++ b/core/src/epicli/cli/engine/UpgradeEngine.py @@ -12,6 +12,7 @@ class UpgradeEngine(Step): def __init__(self, input_data): super().__init__(__name__) self.build_dir = input_data.build_directory + self.ansible_options = {'profile_tasks': getattr(input_data, 'profile_ansible_tasks', False)} self.backup_build_dir = '' self.ansible_command = AnsibleCommand() @@ -52,7 +53,8 @@ def upgrade(self): self.backup_build() # Run Ansible to upgrade infrastructure - with AnsibleRunner(build_dir=self.build_dir, backup_build_dir=self.backup_build_dir) as ansible_runner: + with AnsibleRunner(build_dir=self.build_dir, backup_build_dir=self.backup_build_dir, + ansible_options=self.ansible_options) as ansible_runner: ansible_runner.upgrade() return 0 \ No newline at end of file diff --git a/core/src/epicli/cli/engine/ansible/AnsibleConfigFileCreator.py b/core/src/epicli/cli/engine/ansible/AnsibleConfigFileCreator.py new file mode 100644 index 0000000000..548cd325fa --- /dev/null +++ b/core/src/epicli/cli/engine/ansible/AnsibleConfigFileCreator.py @@ -0,0 +1,55 @@ +import os +from cli.helpers.build_saver import get_ansible_config_file_path, save_ansible_config_file +from cli.helpers.Step import Step +from collections import OrderedDict + +class AnsibleConfigFileCreator(Step): + def __init__(self, ansible_options, cluster_model): + super().__init__(__name__) + self.ansible_options = ansible_options + cluster_name = cluster_model.specification.name + self.ansible_config_file_path = get_ansible_config_file_path(cluster_name) + self.ansible_config_file_settings = OrderedDict() + + def __enter__(self): + super().__enter__() + return self + + def __exit__(self, exc_type, exc_value, traceback): + super().__exit__(exc_type, exc_value, traceback) + + def get_setting(self, section, key): + setting = None + if section in self.ansible_config_file_settings: + setting = self.ansible_config_file_settings[section].get(key) + return setting + + def add_setting(self, section, key, value): + if section not in self.ansible_config_file_settings: + self.ansible_config_file_settings[section] = {} + if key not in self.ansible_config_file_settings[section]: + self.ansible_config_file_settings[section].update({key: value}) + else: + raise TypeError(f"Setting {section}[{key}] already exists") + + def update_setting(self, section, key, value, append=False): + if (section not in self.ansible_config_file_settings or + key not in self.ansible_config_file_settings[section]): + self.add_setting(section, key, value) + else: + if type(self.ansible_config_file_settings[section][key]) is list and append: + self.ansible_config_file_settings[section][key].append(value) + else: + self.ansible_config_file_settings[section][key] = value + + def process_ansible_options(self): + callback_whitelist = [] + if self.ansible_options['profile_tasks']: + callback_whitelist = ['profile_tasks'] + self.add_setting('defaults', 'callback_whitelist', callback_whitelist) + + def create(self): + self.logger.info('Creating ansible.cfg') + self.process_ansible_options() + save_ansible_config_file(self.ansible_config_file_settings, self.ansible_config_file_path) + os.environ["ANSIBLE_CONFIG"] = self.ansible_config_file_path diff --git a/core/src/epicli/cli/engine/ansible/AnsibleInventoryCreator.py b/core/src/epicli/cli/engine/ansible/AnsibleInventoryCreator.py index 4418835379..b20038ded2 100644 --- a/core/src/epicli/cli/engine/ansible/AnsibleInventoryCreator.py +++ b/core/src/epicli/cli/engine/ansible/AnsibleInventoryCreator.py @@ -23,6 +23,7 @@ def __exit__(self, exc_type, exc_value, traceback): # todo: add login for ansible def create(self): + self.logger.info('Creating Ansible inventory') inventory = self.get_inventory() save_inventory(inventory, self.cluster_model) diff --git a/core/src/epicli/cli/engine/ansible/AnsibleRunner.py b/core/src/epicli/cli/engine/ansible/AnsibleRunner.py index 347af57bfc..770c6ce3f8 100644 --- a/core/src/epicli/cli/engine/ansible/AnsibleRunner.py +++ b/core/src/epicli/cli/engine/ansible/AnsibleRunner.py @@ -6,10 +6,12 @@ from cli.engine.ansible.AnsibleCommand import AnsibleCommand from cli.engine.ansible.AnsibleInventoryCreator import AnsibleInventoryCreator +from cli.engine.ansible.AnsibleConfigFileCreator import AnsibleConfigFileCreator from cli.engine.ansible.AnsibleVarsGenerator import AnsibleVarsGenerator from cli.engine.ansible.AnsibleInventoryUpgrade import AnsibleInventoryUpgrade from cli.helpers.Step import Step -from cli.helpers.build_saver import get_inventory_path, get_inventory_path_for_build, get_ansible_path, get_ansible_path_for_build, copy_files_recursively +from cli.helpers.build_saver import (get_inventory_path, get_inventory_path_for_build, get_ansible_path, + get_ansible_path_for_build, copy_files_recursively) from cli.helpers.naming_helpers import to_role_name from cli.helpers.data_loader import DATA_FOLDER_PATH from cli.helpers.Config import Config @@ -18,12 +20,14 @@ class AnsibleRunner(Step): ANSIBLE_PLAYBOOKS_PATH = DATA_FOLDER_PATH + '/common/ansible/playbooks/' - def __init__(self, cluster_model=None, config_docs=None, build_dir=None, backup_build_dir=None): + def __init__(self, cluster_model=None, config_docs=None, build_dir=None, backup_build_dir=None, + ansible_options=None): super().__init__(__name__) self.cluster_model = cluster_model self.config_docs = config_docs self.build_dir = build_dir self.backup_build_dir = backup_build_dir + self.ansible_options = ansible_options self.ansible_command = AnsibleCommand() def __enter__(self): @@ -90,6 +94,10 @@ def apply(self): inventory_creator.create() time.sleep(10) + # create ansible.cfg + ansible_cfg_creator = AnsibleConfigFileCreator(self.ansible_options, self.cluster_model) + ansible_cfg_creator.create() + # generate vars ansible_vars_generator = AnsibleVarsGenerator(inventory_creator=inventory_creator) ansible_vars_generator.generate() @@ -117,6 +125,10 @@ def upgrade(self): inventory_upgrade = AnsibleInventoryUpgrade(self.build_dir, self.backup_build_dir) inventory_upgrade.upgrade() + # create ansible.cfg + ansible_cfg_creator = AnsibleConfigFileCreator(self.ansible_options, self.cluster_model) + ansible_cfg_creator.create() + # generate vars ansible_vars_generator = AnsibleVarsGenerator(inventory_upgrade=inventory_upgrade) ansible_vars_generator.generate() diff --git a/core/src/epicli/cli/epicli.py b/core/src/epicli/cli/epicli.py index fe4df1e163..1da7cfd41b 100644 --- a/core/src/epicli/cli/epicli.py +++ b/core/src/epicli/cli/epicli.py @@ -63,6 +63,7 @@ def debug_level(x): if x < 0 or x > 4: raise argparse.ArgumentTypeError("--debug value should be between 0 and 4") return x + parser.add_argument('--debug', dest='debug', type=debug_level, help='''Set this flag (0..4) to enable debug output where 0 is no debug output and 1..4 is debug output with different verbosity levels: @@ -179,6 +180,9 @@ def apply_parser(subparsers): help='Path to the folder with pre-prepared offline requirements.') sub_parser.add_argument('--vault-password', dest='vault_password', type=str, help='Password that will be used to encrypt build artifacts.') + # developer options + sub_parser.add_argument('--profile-ansible-tasks', dest='profile_ansible_tasks', action="store_true", + help='Enable Ansible profile_tasks plugin for timing tasks.') def run_apply(args): adjust_paths_from_file(args) @@ -213,6 +217,9 @@ def upgrade_parser(subparsers): help="Waits for all pods to be in the 'Ready' state before proceeding to the next step of the K8s upgrade.") sub_parser.add_argument('--offline-requirements', dest='offline_requirements', type=str, required=False, help='Path to the folder with pre-prepared offline requirements.') + # developer options + sub_parser.add_argument('--profile-ansible-tasks', dest='profile_ansible_tasks', action="store_true", + help='Enable Ansible profile_tasks plugin for timing tasks.') def run_upgrade(args): adjust_paths_from_build(args) diff --git a/core/src/epicli/cli/helpers/build_saver.py b/core/src/epicli/cli/helpers/build_saver.py index ec2d63cd29..71b8fc7bda 100644 --- a/core/src/epicli/cli/helpers/build_saver.py +++ b/core/src/epicli/cli/helpers/build_saver.py @@ -45,6 +45,12 @@ def save_inventory(inventory, cluster_model, build_dir=None): save_to_file(file_path, content) +def save_ansible_config_file(ansible_config_file_settings, ansible_config_file_path): + template = load_template_file(types.ANSIBLE, "common", "ansible.cfg") + content = template.render(ansible_config_file_settings=ansible_config_file_settings) + save_to_file(ansible_config_file_path, content) + + # method cleans generated .tf files (not tfstate) def clear_terraform_templates(cluster_name): terraform_dir = get_terraform_path(cluster_name) @@ -94,6 +100,10 @@ def get_inventory_path_for_build(build_directory): return join(inventory, files[0]) +def get_ansible_config_file_path(cluster_name): + return os.path.join(get_build_path(cluster_name), "ansible/ansible.cfg") + + def check_build_output_version(build_directory): if not os.path.exists(build_directory): raise Exception('Build directory does not exist') diff --git a/core/src/epicli/data/common/ansible/ansible.cfg.j2 b/core/src/epicli/data/common/ansible/ansible.cfg.j2 new file mode 100644 index 0000000000..629fa1e3bf --- /dev/null +++ b/core/src/epicli/data/common/ansible/ansible.cfg.j2 @@ -0,0 +1,17 @@ +# +# Ansible configuration file - automated so do not modify +# + +# Ansible will read ANSIBLE_CONFIG, ansible.cfg in the current working directory, +# ansible.cfg in the home directory, or /etc/ansible/ansible.cfg, whichever it finds first. + +# For a full list of available options, run ansible-config list or see the +# documentation: https://docs.ansible.com/ansible/latest/reference_appendices/config.html. + +{% for section in ansible_config_file_settings.keys() -%} +[{{ section }}] +{% for key, value in ansible_config_file_settings[section].items() -%} +{{ key }} = {{ value|join(', ') if (value is sequence and value is not string) else value }} +{% endfor -%} +{% if not loop.last %}{{ '\n' }}{% endif -%} +{% endfor -%} diff --git a/core/src/epicli/data/common/ansible/main.yml.j2 b/core/src/epicli/data/common/ansible/main.yml.j2 deleted file mode 100644 index 5a80ec1f50..0000000000 --- a/core/src/epicli/data/common/ansible/main.yml.j2 +++ /dev/null @@ -1,3 +0,0 @@ -{%- for vars_file in vars_files %} -- include: {{ vars_file }} -{%- endfor %} \ No newline at end of file