Refactor manifest file handling in cli (hitachienergy#3186)

* Create the ManifestReader class for handling configuraton data used by
  the epicli

* Rename all of the manifest variables to input/output_manifest

cli/epicli.py

@@ -8,7 +8,7 @@
 import subprocess
 import sys
 import time
-
+from pathlib import Path
 from typing import List
 
 from cli.licenses import LICENSES
@@ -195,7 +195,7 @@ def apply_parser(subparsers):
     required = sub_parser.add_argument_group('required arguments')
 
     #required
-    required.add_argument('-f', '--file', dest='file', type=str, required=True,
+    required.add_argument('-f', '--input-manifest', dest='input_manifest', type=Path, required=True,
                             help='File with infrastructure/configuration definitions to use.')
 
     #optional
@@ -286,7 +286,7 @@ def upgrade_parser(subparsers):
                             help='Absolute path to directory with build artifacts.')
 
     #optional
-    optional.add_argument('-f', '--file', dest='file', type=str, required=False,
+    optional.add_argument('-f', '--input-manifest', dest='input_manifest', type=str, required=False,
                             help='File with upgraded configuration definitions to use for the components to be upgraded.')
     optional.add_argument('--offline-requirements', dest='offline_requirements', type=str, required=False,
                             help='Path to the folder with pre-prepared offline requirements.')
@@ -356,7 +356,7 @@ def backup_parser(subparsers):
     required = sub_parser.add_argument_group('required arguments')
 
     #required
-    required.add_argument('-f', '--file', dest='file', type=str, required=True,
+    required.add_argument('-f', '--input-manifest', dest='input_manifest', type=str, required=True,
                             help='Backup configuration definition file to use.')
     required.add_argument('-b', '--build', dest='build_directory', type=str, required=True,
                             help='Absolute path to directory with build artifacts.',
@@ -382,7 +382,7 @@ def recovery_parser(subparsers):
     required = sub_parser.add_argument_group('required arguments')
 
     #required
-    required.add_argument('-f', '--file', dest='file', type=str, required=True,
+    required.add_argument('-f', '--input-manifest', dest='input_manifest', type=str, required=True,
                             help='Recovery configuration definition file to use.')
     required.add_argument('-b', '--build', dest='build_directory', type=str, required=True,
                             help='Absolute path to directory with build artifacts.',
@@ -412,13 +412,13 @@ def adjust_paths_from_output_dir():
 
 
 def adjust_paths_from_file(args):
-    if not os.path.isabs(args.file):
-        args.file = os.path.join(os.getcwd(), args.file)
-    if not os.path.isfile(args.file):
+    if not os.path.isabs(args.input_manifest):
+        args.input_manifest = os.path.join(os.getcwd(), args.input_manifest)
+    if not os.path.isfile(args.input_manifest):
         Config().output_dir = os.getcwd()  # Default to working dir so we can at least write logs.
-        raise Exception(f'File "{args.file}" does not exist')
+        raise Exception(f'File "{args.input_manifest}" does not exist')
     if Config().output_dir is None:
-        Config().output_dir = os.path.join(os.path.dirname(args.file), 'build')
+        Config().output_dir = os.path.join(os.path.dirname(args.input_manifest), 'build')
 
 
 def adjust_paths_from_build(args):

cli/src/ansible/AnsibleInventoryUpgrade.py

@@ -1,12 +1,13 @@
-from cli.src.helpers.build_io import (get_inventory_path_for_build,
-                                      load_inventory, load_manifest,
-                                      save_inventory)
+from pathlib import Path
+
+from cli.src.Step import Step
+from cli.src.helpers.build_io import get_inventory_path_for_build, load_inventory, save_inventory
 from cli.src.helpers.data_loader import load_schema_obj, schema_types
 from cli.src.helpers.doc_list_helpers import select_single
 from cli.src.helpers.objdict_helpers import merge_objdict
 from cli.src.models.AnsibleHostModel import AnsibleHostModel
 from cli.src.models.AnsibleInventoryItem import AnsibleInventoryItem
-from cli.src.Step import Step
+from cli.src.schema.ManifestHandler import ManifestHandler
 
 
 class AnsibleInventoryUpgrade(Step):
@@ -16,7 +17,7 @@ def __init__(self, build_dir, backup_build_dir, config_docs):
         self.backup_build_dir = backup_build_dir
         self.cluster_model = None
         self.config_docs = config_docs
-        self.manifest_docs = []
+        self.mhandler: ManifestHandler = ManifestHandler(build_path=Path(build_dir))
 
     def __enter__(self):
         super().__enter__()
@@ -68,8 +69,9 @@ def upgrade(self):
         self.logger.info('Upgrading Ansible inventory')
 
         # load cluster model from manifest
-        self.manifest_docs = load_manifest(self.backup_build_dir)
-        self.cluster_model = select_single(self.manifest_docs, lambda x: x.kind == 'epiphany-cluster')
+        mhandler = ManifestHandler(build_path=Path(self.backup_build_dir))
+        mhandler.read_manifest()
+        self.cluster_model = mhandler.cluster_model
 
         # Merge manifest cluster config with newer defaults
         default_cluster_model = load_schema_obj(schema_types.DEFAULT, self.cluster_model.provider, 'epiphany-cluster')

cli/src/ansible/AnsibleVarsGenerator.py

@@ -13,6 +13,7 @@
 from cli.src.helpers.ObjDict import ObjDict
 from cli.src.helpers.yaml_helpers import dump
 from cli.src.schema.DefaultMerger import DefaultMerger
+from cli.src.schema.ManifestHandler import ManifestHandler
 from cli.src.Step import Step
 from cli.version import VERSION
 
@@ -25,7 +26,6 @@ def __init__(self, inventory_creator=None, inventory_upgrade=None):
         self.inventory_creator = inventory_creator
         self.inventory_upgrade = inventory_upgrade
         self.roles_with_generated_vars = []
-        self.manifest_docs = []
 
         if inventory_creator is not None and inventory_upgrade is None:
             self.cluster_model = inventory_creator.cluster_model
@@ -40,7 +40,7 @@ def __init__(self, inventory_creator=None, inventory_upgrade=None):
                     self.config_docs.append(default)
                 else:
                     self.config_docs.append(config_doc)
-            self.manifest_docs = inventory_upgrade.manifest_docs
+            self.mhandler: ManifestHandler = inventory_upgrade.manifest_docs
         else:
             raise Exception('Invalid AnsibleVarsGenerator configuration')
 
@@ -120,11 +120,11 @@ def write_role_vars(self, ansible_dir, role, document, vars_file_name='main.yml'
 
     def write_role_manifest_vars(self, ansible_dir, role, kind):
         try:
-            cluster_model = select_single(self.manifest_docs, lambda x: x.kind == 'epiphany-cluster')
-        except ExpectedSingleResultException:
+            cluster_model = self.mhandler['epiphany-cluster']
+        except IndexError:
             return  # skip
 
-        document = select_first(self.manifest_docs, lambda x: x.kind == kind)
+        document = self.mhandler[kind]
         if document is None:
             # If there is no document provided by the user, then fallback to defaults
             document = load_schema_obj(schema_types.DEFAULT, 'common', kind)
@@ -205,10 +205,10 @@ def get_shared_config_from_manifest(self):
         # Reuse shared config from existing manifest
         # Shared config contains the use_ha_control_plane flag which is required during upgrades
 
-        cluster_model = select_single(self.manifest_docs, lambda x: x.kind == 'epiphany-cluster')
+        cluster_model = self.mhandler['epiphany-cluster']
 
         try:
-            shared_config_doc = select_single(self.manifest_docs, lambda x: x.kind == 'configuration/shared-config')
+            shared_config_doc = self.mhandler['configuration/shared-config']
             shared_config_doc['provider'] = cluster_model['provider']
         except ExpectedSingleResultException:
             # If there is no shared-config doc inside the manifest file, this is probably a v0.3 cluster