[2.0.1] Add manifest file parsing (#3105) (#3130)

* Add manifest file parsing (#3105)

* Add `-m/--manifest` flag to accept manifest.yml produced by
  `epicli init/prepare`

* Add `-v/--verbose` mode for printing out parsed manifest data

* Add ManifestReader class used for paring the manifest.yml file

* Move src/command/*.py to debian/redhat subdirs where needed

ansible/playbooks/roles/repository/defaults/main.yml

@@ -1,4 +1,5 @@
 ---
 download_requirements_dir: "/var/tmp/epi-download-requirements"
-download_requirements_script: "{{ download_requirements_dir }}/download-requirements.py"
 download_requirements_flag: "{{ download_requirements_dir }}/download-requirements-done.flag"
+download_requirements_manifest: "{{ download_requirements_dir }}/manifest.yml"
+download_requirements_script: "{{ download_requirements_dir }}/download-requirements.py"

ansible/playbooks/roles/repository/files/download-requirements/src/command/toolchain.py

@@ -1,17 +1,17 @@
 import logging
 from typing import Dict
 
-from src.command.apt import Apt
-from src.command.apt_cache import AptCache
-from src.command.apt_key import AptKey
 from src.command.crane import Crane
-from src.command.dnf_repoquery import DnfRepoquery
-from src.command.rpm import Rpm
+from src.command.debian.apt import Apt
+from src.command.debian.apt_cache import AptCache
+from src.command.debian.apt_key import AptKey
+from src.command.redhat.dnf import Dnf
+from src.command.redhat.dnf_config_manager import DnfConfigManager
+from src.command.redhat.dnf_download import DnfDownload
+from src.command.redhat.dnf_repoquery import DnfRepoquery
+from src.command.redhat.rpm import Rpm
 from src.command.tar import Tar
 from src.command.wget import Wget
-from src.command.dnf import Dnf
-from src.command.dnf_config_manager import DnfConfigManager
-from src.command.dnf_download import DnfDownload
 from src.config.os_type import OSFamily
 
 

ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py

@@ -4,8 +4,9 @@
 from itertools import chain
 from os import uname
 from pathlib import Path
-from typing import List
+from typing import Any, Dict, List
 
+from src.config.manifest_reader import ManifestReader
 from src.config.os_type import OSArch, OSConfig, OSType, SUPPORTED_OS_TYPES
 from src.error import CriticalError
 
@@ -17,6 +18,7 @@ def __init__(self, argv: List[str]):
         self.dest_files: Path
         self.dest_grafana_dashboards: Path
         self.dest_images: Path
+        self.dest_manifest: Path
         self.dest_packages: Path
         self.distro_subdir: Path
         self.is_log_file_enabled: bool
@@ -27,24 +29,26 @@ def __init__(self, argv: List[str]):
         self.repo_path: Path
         self.repos_backup_file: Path
         self.reqs_path: Path
-        self.rerun: bool
+        self.rerun: bool = False
         self.retries: int
         self.script_path: Path
+        self.verbose_mode: bool
         self.was_backup_created: bool = False
 
         self.__add_args(argv)
 
         if not self.rerun:
             self.__log_info_summary()
 
+        self.__LINE_SIZE: int = 50  # used in printing
+
     def __log_info_summary(self):
         """
         Helper function for printing all parsed arguments
         """
 
         lines: List[str] = ['Info summary:']
-        LINE_SIZE: int = 50
-        lines.append('-' * LINE_SIZE)
+        lines.append('-' * self.__LINE_SIZE)
 
         lines.append(f'OS Arch: {self.os_arch.value}')
         lines.append(f'OS Type: {self.os_type.os_name}')
@@ -56,12 +60,16 @@ def __log_info_summary(self):
         lines.append(f'- packages:           {str(self.dest_packages)}')
         lines.append(f'Repos backup file: {str(self.repos_backup_file)}')
 
+        if self.dest_manifest:
+            lines.append(f'Manifest used: {str(self.dest_manifest)}')
+
         if self.is_log_file_enabled:
             lines.append(f'Log file location: {str(self.log_file.absolute())}')
 
+        lines.append(f'Verbose mode: {self.verbose_mode}')
         lines.append(f'Retries count: {self.retries}')
 
-        lines.append('-' * LINE_SIZE)
+        lines.append('-' * self.__LINE_SIZE)
 
         logging.info('\n'.join(lines))
 
@@ -93,6 +101,12 @@ def __create_parser(self) -> ArgumentParser:
         parser.add_argument('--no-logfile', action='store_true', dest='no_logfile',
                             help='no logfile will be created')
 
+        parser.add_argument('--verbose', '-v',  action='store_true', dest='verbose',
+                            help='more verbose output will be provided')
+
+        parser.add_argument('--manifest', '-m', metavar='MANIFEST_PATH', type=Path, action='store', dest='manifest',
+                            help='manifest file generated by epicli')
+
         # offline mode rerun options:
         parser.add_argument('--rerun', action='store_true', dest='rerun',
                             default=False, help=SUPPRESS)
@@ -195,7 +209,44 @@ def __add_args(self, argv: List[str]):
         self.repos_backup_file = Path(args['repos_backup_file'])
         self.retries = args['retries']
         self.is_log_file_enabled = False if args['no_logfile'] else True
+        self.dest_manifest = args['manifest'] or None
+        self.verbose_mode = args['verbose']
 
         # offline mode
         self.rerun = args['rerun']
         self.pyyaml_installed = args['pyyaml_installed']
+
+    def __print_parsed_manifest_data(self, output: Dict[str, Any]):
+        lines: List[str] = ['Manifest summary:']
+
+        lines.append('-' * self.__LINE_SIZE)
+
+        lines.append('Components detected:')
+        for component in output['detected-components']:
+            lines.append(f'- {component}')
+
+        lines.append('')
+
+        lines.append('Features detected:')
+        for feature in output['detected-features']:
+            lines.append(f'- {feature}')
+
+        lines.append('-' * self.__LINE_SIZE)
+
+        logging.info('\n'.join(lines))
+
+    def read_manifest(self, requirements: Dict[str, Any]):
+        """
+        Construct ManifestReader and parse only required data.
+        Not needed entries will be removed from the `requirements`
+
+        :param requirements: parsed requirements which will be filtered based on the manifest output
+        """
+        if not self.dest_manifest:
+            return
+
+        mreader = ManifestReader(self.dest_manifest)
+        output = mreader.parse_manifest()
+
+        if self.verbose_mode:
+            self.__print_parsed_manifest_data(output)

ansible/playbooks/roles/repository/files/download-requirements/src/config/manifest_reader.py

@@ -0,0 +1,78 @@
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Set
+
+import yaml
+
+from src.error import CriticalError
+
+
+def load_yaml_file_all(filename: Path) -> List[Any]:
+    try:
+        with open(filename, encoding="utf-8") as req_handler:
+            return list(yaml.safe_load_all(req_handler))
+    except yaml.YAMLError as yaml_err:
+        raise CriticalError(f'Failed loading: `{yaml_err}`') from yaml_err
+    except Exception as err:
+        raise CriticalError(f'Failed loading: `{filename}`') from err
+
+
+def load_yaml_file(filename: Path) -> Any:
+    return load_yaml_file_all(filename)[0]
+
+
+class ManifestReader:
+    """
+    Load the manifest file and call defined parser methods to process required documents.
+    Main running method is :func:`~manifest_reader.ManifestReader.parse_manifest` which returns formatted manifest output.
+    """
+
+    def __init__(self, dest_manifest: Path):
+        self.__dest_manifest = dest_manifest
+        self.__detected_components: Set = set()
+        self.__detected_features: Set = set()
+
+    def __parse_cluster_info(self, cluster_doc: Dict):
+        """
+        Parse `epiphany-cluster` document and extract only used components.
+
+        :param cluster_doc: handler to a `epiphany-cluster` document
+        """
+        components = cluster_doc['specification']['components']
+        for component in components:
+            if components[component]['count'] > 0:
+                self.__detected_components.add(component)
+
+    def __parse_feature_mappings_info(self, feature_mappings_doc: Dict):
+        """
+        Parse `configuration/feature-mappings` document and extract only used features (based on `epiphany-cluster` doc).
+
+        :param feature_mappings_doc: handler to a `configuration/feature-mappings` document
+        """
+        mappings = feature_mappings_doc['specification']['mappings']
+        for mapping in mappings.keys() & self.__detected_components:
+            for feature in mappings[mapping]:
+                self.__detected_features.add(feature)
+
+    def parse_manifest(self) -> Dict[str, Any]:
+        """
+        Load the manifest file, call parsers on required docs and return formatted output.
+        """
+        parse_doc: Dict[str, Callable] = {
+            'epiphany-cluster':               self.__parse_cluster_info,
+            'configuration/feature-mappings': self.__parse_feature_mappings_info
+        }
+
+        parsed_docs: Set[str] = set()
+        for manifest_doc in load_yaml_file_all(self.__dest_manifest):
+            try:
+                kind: str = manifest_doc['kind']
+                parse_doc[kind](manifest_doc)
+                parsed_docs.add(kind)
+            except KeyError:
+                pass
+
+        if len(parsed_docs) != len(parse_doc.keys()):
+            raise CriticalError(f'ManifestReader - could not find documents: {parsed_docs ^ parse_doc.keys()}')
+
+        return {'detected-components': sorted(list(self.__detected_components)),
+                'detected-features': sorted(list(self.__detected_features))}

ansible/playbooks/roles/repository/files/download-requirements/src/mode/base_mode.py

@@ -4,25 +4,14 @@
 from pathlib import Path
 from typing import Any, Dict
 
-import yaml
-
 from src.command.toolchain import Toolchain, TOOLCHAINS
 from src.config.config import Config, OSArch
+from src.config.manifest_reader import load_yaml_file
 from src.crypt import SHA_ALGORITHMS
 from src.downloader import Downloader
 from src.error import CriticalError, ChecksumMismatch
 
 
-def load_yaml_file(filename: Path) -> Any:
-    try:
-        with open(filename, encoding="utf-8") as req_handler:
-            return yaml.safe_load(req_handler)
-    except yaml.YAMLError as yaml_err:
-        raise CriticalError(f'Failed loading: `{yaml_err}`') from yaml_err
-    except Exception as err:
-        raise CriticalError(f'Failed loading: `{filename}`') from err
-
-
 class BaseMode:
     """
     An abstract class for running specific operations on target OS.
@@ -35,6 +24,7 @@ def __init__(self, config: Config):
         self._repositories: Dict[str, Dict] = self.__parse_repositories()
         self._requirements: Dict[str, Any] = self.__parse_requirements()
         self._tools: Toolchain = TOOLCHAINS[self._cfg.os_type.os_family](self._cfg.retries)
+        self._cfg.read_manifest(self._requirements)
 
     def __parse_repositories(self) -> Dict[str, Dict]:
         """

ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt.py → ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt.py

@@ -1,6 +1,6 @@
 from tests.mocks.command_run_mock import CommandRunMock
 
-from src.command.apt import Apt
+from src.command.debian.apt import Apt
 
 
 def test_interface_update(mocker):

ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt_cache.py → ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt_cache.py

@@ -1,6 +1,6 @@
 from tests.mocks.command_run_mock import CommandRunMock
 
-from src.command.apt_cache import AptCache
+from src.command.debian.apt_cache import AptCache
 
 
 def test_interface_get_package_dependencies(mocker):

ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt_key.py → ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt_key.py

@@ -2,7 +2,7 @@
 
 from tests.mocks.command_run_mock import CommandRunMock
 
-from src.command.apt_key import AptKey
+from src.command.debian.apt_key import AptKey
 
 
 def test_interface_add(mocker):

ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf.py → ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf.py

@@ -1,4 +1,4 @@
-from src.command.dnf import Dnf
+from src.command.redhat.dnf import Dnf
 
 from tests.mocks.command_run_mock import CommandRunMock
 

ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_config_manager.py → ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_config_manager.py

@@ -1,6 +1,6 @@
 from tests.mocks.command_run_mock import CommandRunMock
 
-from src.command.dnf_config_manager import DnfConfigManager
+from src.command.redhat.dnf_config_manager import DnfConfigManager
 
 
 def test_interface_add_repo(mocker):

ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_download.py → ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_download.py

@@ -2,7 +2,7 @@
 
 from tests.mocks.command_run_mock import CommandRunMock
 
-from src.command.dnf_download import DnfDownload
+from src.command.redhat.dnf_download import DnfDownload
 
 
 def test_interface_download_packages(mocker):

ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_repoquery.py → ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_repoquery.py

@@ -1,6 +1,6 @@
 from tests.mocks.command_run_mock import CommandRunMock
 
-from src.command.dnf_repoquery import DnfRepoquery
+from src.command.redhat.dnf_repoquery import DnfRepoquery
 
 
 def test_interface_query(mocker):

ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_rpm.py → ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_rpm.py

@@ -1,6 +1,6 @@
 from tests.mocks.command_run_mock import CommandRunMock
 
-from src.command.rpm import Rpm
+from src.command.redhat.rpm import Rpm
 
 
 def test_interface_is_package_installed(mocker):

ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py

@@ -0,0 +1,30 @@
+import logging
+from pathlib import Path
+
+import yaml
+
+from src.config.config import Config
+from tests.data.config import EXPECTED_VERBOSE_OUTPUT
+from tests.data.manifest_reader import INPUT_MANIFEST_FEATURE_MAPPINGS
+
+
+def test_manifest_verbose_output(mocker, caplog):
+    ''' Check output produced when running download-requirements script with the `-v|--verbose` flag and with provided `-m|--manifest` '''
+
+    mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_MANIFEST_FEATURE_MAPPINGS))
+    caplog.set_level(logging.INFO)
+
+    # mock Config's init methods:
+    Config._Config__add_args = lambda *args: None
+    Config._Config__log_info_summary = lambda *args: None
+
+    config = Config([])
+
+    # mock required config data:
+    config.dest_manifest = Path('/some/path')
+    config.verbose_mode = True
+    config.read_manifest({})
+
+    log_output = f'\n{"".join(caplog.messages)}\n'
+
+    assert log_output == EXPECTED_VERBOSE_OUTPUT

ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py

@@ -0,0 +1,13 @@
+from pathlib import Path
+
+import yaml
+
+from src.config.manifest_reader import ManifestReader
+from tests.data.manifest_reader import EXPECTED_FEATURE_MAPPINGS, INPUT_MANIFEST_FEATURE_MAPPINGS
+
+def test_parse_manifest(mocker):
+    ''' Check manifest file parsing '''
+    mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_MANIFEST_FEATURE_MAPPINGS))
+
+    mreader = ManifestReader(Path('/some/path'))
+    assert mreader.parse_manifest() == EXPECTED_FEATURE_MAPPINGS