Enable Ansible profile_tasks plugin for timing tasks

CHANGELOG-0.7.md

@@ -0,0 +1,9 @@
+# Changelog 0.7
+
+## [0.7.0] 2020-0X-XX
+
+### Added
+
+#### General
+
+- [#811](https://github.com/epiphany-platform/epiphany/issues/811) - Measure execution time of Ansible tasks

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 Reference for actual cluster component versions can be found [here](docs/home/COMPONENTS.md)
 
+- [CHANGELOG-0.7.x](./CHANGELOG-0.7.md)
 - [CHANGELOG-0.6.x](./CHANGELOG-0.6.md)
 - [CHANGELOG-0.5.x](./CHANGELOG-0.5.md)
 - [CHANGELOG-0.4.x](./CHANGELOG-0.4.md)

core/src/epicli/cli/engine/ApplyEngine.py

@@ -20,8 +20,9 @@
 class ApplyEngine(Step):
     def __init__(self, input_data):
         self.file = input_data.file
-        self.skip_infrastructure = input_data.no_infra if hasattr(input_data, 'no_infra') else False
-        self.skip_config = input_data.skip_config if hasattr(input_data, 'skip_config') else False
+        self.skip_infrastructure = getattr(input_data, 'no_infra', False)
+        self.skip_config = getattr(input_data, 'skip_config', False)
+        self.ansible_options = {'profile_tasks': getattr(input_data, 'profile_ansible_tasks', False)}
         self.logger = Log(__name__)
 
         self.cluster_model = None
@@ -147,7 +148,7 @@ def apply(self):
 
         # Run Ansible to provision infrastructure
         if not(self.skip_config):
-            with AnsibleRunner(self.cluster_model, docs) as ansible_runner:
+            with AnsibleRunner(self.cluster_model, docs, ansible_options=self.ansible_options) as ansible_runner:
                 ansible_runner.apply()
 
         return 0

core/src/epicli/cli/engine/UpgradeEngine.py

@@ -12,6 +12,7 @@ class UpgradeEngine(Step):
     def __init__(self, input_data):
         super().__init__(__name__)
         self.build_dir = input_data.build_directory
+        self.ansible_options = {'profile_tasks': getattr(input_data, 'profile_ansible_tasks', False)}
         self.backup_build_dir = ''
         self.ansible_command = AnsibleCommand()
 
@@ -52,7 +53,8 @@ def upgrade(self):
         self.backup_build()
 
         # Run Ansible to upgrade infrastructure
-        with AnsibleRunner(build_dir=self.build_dir, backup_build_dir=self.backup_build_dir) as ansible_runner:
+        with AnsibleRunner(build_dir=self.build_dir, backup_build_dir=self.backup_build_dir,
+                           ansible_options=self.ansible_options) as ansible_runner:
             ansible_runner.upgrade()
 
         return 0      

core/src/epicli/cli/engine/ansible/AnsibleConfigFileCreator.py

@@ -0,0 +1,55 @@
+import os
+from cli.helpers.build_saver import get_ansible_config_file_path, save_ansible_config_file
+from cli.helpers.Step import Step
+from collections import OrderedDict
+
+class AnsibleConfigFileCreator(Step):
+    def __init__(self, ansible_options, cluster_model):
+        super().__init__(__name__)
+        self.ansible_options = ansible_options
+        cluster_name = cluster_model.specification.name
+        self.ansible_config_file_path = get_ansible_config_file_path(cluster_name)
+        self.ansible_config_file_settings = OrderedDict()
+
+    def __enter__(self):
+        super().__enter__()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        super().__exit__(exc_type, exc_value, traceback)
+
+    def get_setting(self, section, key):
+        setting = None
+        if section in self.ansible_config_file_settings:
+            setting = self.ansible_config_file_settings[section].get(key)
+        return setting
+
+    def add_setting(self, section, key, value):
+        if section not in self.ansible_config_file_settings:
+            self.ansible_config_file_settings[section] = {}
+        if key not in self.ansible_config_file_settings[section]:
+            self.ansible_config_file_settings[section].update({key: value})
+        else:
+            raise TypeError(f"Setting {section}[{key}] already exists")
+
+    def update_setting(self, section, key, value, append=False):
+        if (section not in self.ansible_config_file_settings or
+            key not in self.ansible_config_file_settings[section]):
+            self.add_setting(section, key, value)
+        else:
+            if type(self.ansible_config_file_settings[section][key]) is list and append:
+                self.ansible_config_file_settings[section][key].append(value)
+            else:
+                self.ansible_config_file_settings[section][key] = value
+
+    def process_ansible_options(self):
+        callback_whitelist = []
+        if self.ansible_options['profile_tasks']:
+            callback_whitelist = ['profile_tasks']
+        self.add_setting('defaults', 'callback_whitelist', callback_whitelist)
+
+    def create(self):
+        self.logger.info('Creating ansible.cfg')
+        self.process_ansible_options()
+        save_ansible_config_file(self.ansible_config_file_settings, self.ansible_config_file_path)
+        os.environ["ANSIBLE_CONFIG"] = self.ansible_config_file_path

core/src/epicli/cli/engine/ansible/AnsibleInventoryCreator.py

@@ -23,6 +23,7 @@ def __exit__(self, exc_type, exc_value, traceback):
 
     # todo: add login for ansible
     def create(self):
+        self.logger.info('Creating Ansible inventory')
         inventory = self.get_inventory()
         save_inventory(inventory, self.cluster_model)
 

core/src/epicli/cli/engine/ansible/AnsibleRunner.py

@@ -6,10 +6,12 @@
 
 from cli.engine.ansible.AnsibleCommand import AnsibleCommand
 from cli.engine.ansible.AnsibleInventoryCreator import AnsibleInventoryCreator
+from cli.engine.ansible.AnsibleConfigFileCreator import AnsibleConfigFileCreator
 from cli.engine.ansible.AnsibleVarsGenerator import AnsibleVarsGenerator
 from cli.engine.ansible.AnsibleInventoryUpgrade import AnsibleInventoryUpgrade
 from cli.helpers.Step import Step
-from cli.helpers.build_saver import get_inventory_path, get_inventory_path_for_build, get_ansible_path, get_ansible_path_for_build, copy_files_recursively
+from cli.helpers.build_saver import (get_inventory_path, get_inventory_path_for_build, get_ansible_path,
+    get_ansible_path_for_build, copy_files_recursively)
 from cli.helpers.naming_helpers import to_role_name
 from cli.helpers.data_loader import DATA_FOLDER_PATH
 from cli.helpers.Config import Config
@@ -18,12 +20,14 @@
 class AnsibleRunner(Step):
     ANSIBLE_PLAYBOOKS_PATH = DATA_FOLDER_PATH + '/common/ansible/playbooks/'
 
-    def __init__(self, cluster_model=None, config_docs=None, build_dir=None, backup_build_dir=None):
+    def __init__(self, cluster_model=None, config_docs=None, build_dir=None, backup_build_dir=None,
+                 ansible_options=None):
         super().__init__(__name__)
         self.cluster_model = cluster_model
         self.config_docs = config_docs
         self.build_dir = build_dir
         self.backup_build_dir = backup_build_dir
+        self.ansible_options = ansible_options
         self.ansible_command = AnsibleCommand()
 
     def __enter__(self):
@@ -90,6 +94,10 @@ def apply(self):
         inventory_creator.create()
         time.sleep(10)
 
+        # create ansible.cfg
+        ansible_cfg_creator = AnsibleConfigFileCreator(self.ansible_options, self.cluster_model)
+        ansible_cfg_creator.create()
+
         # generate vars
         ansible_vars_generator = AnsibleVarsGenerator(inventory_creator=inventory_creator)      
         ansible_vars_generator.generate()
@@ -117,6 +125,10 @@ def upgrade(self):
         inventory_upgrade = AnsibleInventoryUpgrade(self.build_dir, self.backup_build_dir)
         inventory_upgrade.upgrade()
 
+        # create ansible.cfg
+        ansible_cfg_creator = AnsibleConfigFileCreator(self.ansible_options, self.cluster_model)
+        ansible_cfg_creator.create()
+
         # generate vars
         ansible_vars_generator = AnsibleVarsGenerator(inventory_upgrade=inventory_upgrade)      
         ansible_vars_generator.generate()        

core/src/epicli/cli/epicli.py

@@ -63,6 +63,7 @@ def debug_level(x):
         if x < 0 or x > 4:
             raise argparse.ArgumentTypeError("--debug value should be between 0 and 4")
         return x
+
     parser.add_argument('--debug', dest='debug', type=debug_level,
                         help='''Set this flag (0..4) to enable debug output where 0 is no
 debug output and 1..4 is debug output with different verbosity levels:
@@ -179,6 +180,9 @@ def apply_parser(subparsers):
                             help='Path to the folder with pre-prepared offline requirements.')    
     sub_parser.add_argument('--vault-password', dest='vault_password', type=str,
                             help='Password that will be used to encrypt build artifacts.')
+    # developer options
+    sub_parser.add_argument('--profile-ansible-tasks', dest='profile_ansible_tasks', action="store_true",
+                            help='Enable Ansible profile_tasks plugin for timing tasks.')
 
     def run_apply(args):
         adjust_paths_from_file(args)
@@ -213,6 +217,9 @@ def upgrade_parser(subparsers):
                             help="Waits for all pods to be in the 'Ready' state before proceeding to the next step of the K8s upgrade.")
     sub_parser.add_argument('--offline-requirements', dest='offline_requirements', type=str, required=False,
                             help='Path to the folder with pre-prepared offline requirements.')
+    # developer options
+    sub_parser.add_argument('--profile-ansible-tasks', dest='profile_ansible_tasks', action="store_true",
+                            help='Enable Ansible profile_tasks plugin for timing tasks.')
 
     def run_upgrade(args):
         adjust_paths_from_build(args)

core/src/epicli/cli/helpers/build_saver.py

@@ -45,6 +45,12 @@ def save_inventory(inventory, cluster_model, build_dir=None):
     save_to_file(file_path, content)
 
 
+def save_ansible_config_file(ansible_config_file_settings, ansible_config_file_path):
+    template = load_template_file(types.ANSIBLE, "common", "ansible.cfg")
+    content = template.render(ansible_config_file_settings=ansible_config_file_settings)
+    save_to_file(ansible_config_file_path, content)
+
+
 # method cleans generated .tf files (not tfstate)
 def clear_terraform_templates(cluster_name):
     terraform_dir = get_terraform_path(cluster_name)
@@ -94,6 +100,10 @@ def get_inventory_path_for_build(build_directory):
         return join(inventory, files[0]) 
 
 
+def get_ansible_config_file_path(cluster_name):
+    return os.path.join(get_build_path(cluster_name), "ansible/ansible.cfg")
+
+
 def check_build_output_version(build_directory):
     if not os.path.exists(build_directory):
         raise Exception('Build directory does not exist')

core/src/epicli/data/common/ansible/ansible.cfg.j2

@@ -0,0 +1,17 @@
+#
+# Ansible configuration file - automated so do not modify
+#
+
+# Ansible will read ANSIBLE_CONFIG, ansible.cfg in the current working directory,
+# ansible.cfg in the home directory, or /etc/ansible/ansible.cfg, whichever it finds first.
+
+# For a full list of available options, run ansible-config list or see the
+# documentation: https://docs.ansible.com/ansible/latest/reference_appendices/config.html.
+
+{% for section in ansible_config_file_settings.keys() -%}
+[{{ section }}]
+{%   for key, value in ansible_config_file_settings[section].items() -%}
+{{ key }} = {{ value|join(', ') if (value is sequence and value is not string) else value }}
+{%   endfor -%}
+{%   if not loop.last %}{{ '\n' }}{% endif -%}
+{% endfor -%}