From f309615c448bb0d49d294dbb9d1cc4bdda2c3d9e Mon Sep 17 00:00:00 2001 From: sbbroot <86356638+sbbroot@users.noreply.github.com> Date: Tue, 17 May 2022 10:01:31 +0200 Subject: [PATCH] [2.0.1] Add manifest file parsing (#3105) (#3130) * Add manifest file parsing (#3105) * Add `-m/--manifest` flag to accept manifest.yml produced by `epicli init/prepare` * Add `-v/--verbose` mode for printing out parsed manifest data * Add ManifestReader class used for paring the manifest.yml file * Move src/command/*.py to debian/redhat subdirs where needed --- .../roles/repository/defaults/main.yml | 3 +- .../src/command/{ => debian}/apt.py | 0 .../src/command/{ => debian}/apt_cache.py | 0 .../src/command/{ => debian}/apt_key.py | 0 .../src/command/{ => redhat}/dnf.py | 0 .../{ => redhat}/dnf_config_manager.py | 0 .../src/command/{ => redhat}/dnf_download.py | 0 .../src/command/{ => redhat}/dnf_repoquery.py | 0 .../src/command/{ => redhat}/rpm.py | 0 .../src/command/toolchain.py | 16 +- .../src/config/config.py | 61 +++++++- .../src/config/manifest_reader.py | 78 ++++++++++ .../src/mode/base_mode.py | 14 +- .../tests/command/{ => debian}/test_apt.py | 2 +- .../command/{ => debian}/test_apt_cache.py | 2 +- .../command/{ => debian}/test_apt_key.py | 2 +- .../tests/command/{ => redhat}/test_dnf.py | 2 +- .../{ => redhat}/test_dnf_config_manager.py | 2 +- .../command/{ => redhat}/test_dnf_download.py | 2 +- .../{ => redhat}/test_dnf_repoquery.py | 2 +- .../tests/command/{ => redhat}/test_rpm.py | 2 +- .../tests/config/test_config.py | 30 ++++ .../tests/config/test_manifest_reader.py | 13 ++ .../tests/data/config.py | 22 +++ .../tests/data/manifest_reader.py | 142 ++++++++++++++++++ .../tasks/copy-download-requirements.yml | 5 + .../tasks/download-requirements.yml | 4 +- 27 files changed, 369 insertions(+), 35 deletions(-) rename ansible/playbooks/roles/repository/files/download-requirements/src/command/{ => debian}/apt.py (100%) rename ansible/playbooks/roles/repository/files/download-requirements/src/command/{ => debian}/apt_cache.py (100%) rename ansible/playbooks/roles/repository/files/download-requirements/src/command/{ => debian}/apt_key.py (100%) rename ansible/playbooks/roles/repository/files/download-requirements/src/command/{ => redhat}/dnf.py (100%) rename ansible/playbooks/roles/repository/files/download-requirements/src/command/{ => redhat}/dnf_config_manager.py (100%) rename ansible/playbooks/roles/repository/files/download-requirements/src/command/{ => redhat}/dnf_download.py (100%) rename ansible/playbooks/roles/repository/files/download-requirements/src/command/{ => redhat}/dnf_repoquery.py (100%) rename ansible/playbooks/roles/repository/files/download-requirements/src/command/{ => redhat}/rpm.py (100%) create mode 100644 ansible/playbooks/roles/repository/files/download-requirements/src/config/manifest_reader.py rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => debian}/test_apt.py (97%) rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => debian}/test_apt_cache.py (93%) rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => debian}/test_apt_key.py (88%) rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => redhat}/test_dnf.py (97%) rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => redhat}/test_dnf_config_manager.py (93%) rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => redhat}/test_dnf_download.py (94%) rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => redhat}/test_dnf_repoquery.py (96%) rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => redhat}/test_rpm.py (96%) create mode 100644 ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py create mode 100644 ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py create mode 100644 ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py create mode 100644 ansible/playbooks/roles/repository/files/download-requirements/tests/data/manifest_reader.py diff --git a/ansible/playbooks/roles/repository/defaults/main.yml b/ansible/playbooks/roles/repository/defaults/main.yml index 5a0a7db45d..95b9f60c6b 100644 --- a/ansible/playbooks/roles/repository/defaults/main.yml +++ b/ansible/playbooks/roles/repository/defaults/main.yml @@ -1,4 +1,5 @@ --- download_requirements_dir: "/var/tmp/epi-download-requirements" -download_requirements_script: "{{ download_requirements_dir }}/download-requirements.py" download_requirements_flag: "{{ download_requirements_dir }}/download-requirements-done.flag" +download_requirements_manifest: "{{ download_requirements_dir }}/manifest.yml" +download_requirements_script: "{{ download_requirements_dir }}/download-requirements.py" diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/apt.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/debian/apt.py similarity index 100% rename from ansible/playbooks/roles/repository/files/download-requirements/src/command/apt.py rename to ansible/playbooks/roles/repository/files/download-requirements/src/command/debian/apt.py diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/apt_cache.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/debian/apt_cache.py similarity index 100% rename from ansible/playbooks/roles/repository/files/download-requirements/src/command/apt_cache.py rename to ansible/playbooks/roles/repository/files/download-requirements/src/command/debian/apt_cache.py diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/apt_key.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/debian/apt_key.py similarity index 100% rename from ansible/playbooks/roles/repository/files/download-requirements/src/command/apt_key.py rename to ansible/playbooks/roles/repository/files/download-requirements/src/command/debian/apt_key.py diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf.py similarity index 100% rename from ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf.py rename to ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf.py diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_config_manager.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf_config_manager.py similarity index 100% rename from ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_config_manager.py rename to ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf_config_manager.py diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_download.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf_download.py similarity index 100% rename from ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_download.py rename to ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf_download.py diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_repoquery.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf_repoquery.py similarity index 100% rename from ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_repoquery.py rename to ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf_repoquery.py diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/rpm.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/rpm.py similarity index 100% rename from ansible/playbooks/roles/repository/files/download-requirements/src/command/rpm.py rename to ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/rpm.py diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/toolchain.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/toolchain.py index 5aba587f13..7c2488c175 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/command/toolchain.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/command/toolchain.py @@ -1,17 +1,17 @@ import logging from typing import Dict -from src.command.apt import Apt -from src.command.apt_cache import AptCache -from src.command.apt_key import AptKey from src.command.crane import Crane -from src.command.dnf_repoquery import DnfRepoquery -from src.command.rpm import Rpm +from src.command.debian.apt import Apt +from src.command.debian.apt_cache import AptCache +from src.command.debian.apt_key import AptKey +from src.command.redhat.dnf import Dnf +from src.command.redhat.dnf_config_manager import DnfConfigManager +from src.command.redhat.dnf_download import DnfDownload +from src.command.redhat.dnf_repoquery import DnfRepoquery +from src.command.redhat.rpm import Rpm from src.command.tar import Tar from src.command.wget import Wget -from src.command.dnf import Dnf -from src.command.dnf_config_manager import DnfConfigManager -from src.command.dnf_download import DnfDownload from src.config.os_type import OSFamily diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py b/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py index b4c9d53b43..b6dde5b00d 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py @@ -4,8 +4,9 @@ from itertools import chain from os import uname from pathlib import Path -from typing import List +from typing import Any, Dict, List +from src.config.manifest_reader import ManifestReader from src.config.os_type import OSArch, OSConfig, OSType, SUPPORTED_OS_TYPES from src.error import CriticalError @@ -17,6 +18,7 @@ def __init__(self, argv: List[str]): self.dest_files: Path self.dest_grafana_dashboards: Path self.dest_images: Path + self.dest_manifest: Path self.dest_packages: Path self.distro_subdir: Path self.is_log_file_enabled: bool @@ -27,9 +29,10 @@ def __init__(self, argv: List[str]): self.repo_path: Path self.repos_backup_file: Path self.reqs_path: Path - self.rerun: bool + self.rerun: bool = False self.retries: int self.script_path: Path + self.verbose_mode: bool self.was_backup_created: bool = False self.__add_args(argv) @@ -37,14 +40,15 @@ def __init__(self, argv: List[str]): if not self.rerun: self.__log_info_summary() + self.__LINE_SIZE: int = 50 # used in printing + def __log_info_summary(self): """ Helper function for printing all parsed arguments """ lines: List[str] = ['Info summary:'] - LINE_SIZE: int = 50 - lines.append('-' * LINE_SIZE) + lines.append('-' * self.__LINE_SIZE) lines.append(f'OS Arch: {self.os_arch.value}') lines.append(f'OS Type: {self.os_type.os_name}') @@ -56,12 +60,16 @@ def __log_info_summary(self): lines.append(f'- packages: {str(self.dest_packages)}') lines.append(f'Repos backup file: {str(self.repos_backup_file)}') + if self.dest_manifest: + lines.append(f'Manifest used: {str(self.dest_manifest)}') + if self.is_log_file_enabled: lines.append(f'Log file location: {str(self.log_file.absolute())}') + lines.append(f'Verbose mode: {self.verbose_mode}') lines.append(f'Retries count: {self.retries}') - lines.append('-' * LINE_SIZE) + lines.append('-' * self.__LINE_SIZE) logging.info('\n'.join(lines)) @@ -93,6 +101,12 @@ def __create_parser(self) -> ArgumentParser: parser.add_argument('--no-logfile', action='store_true', dest='no_logfile', help='no logfile will be created') + parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', + help='more verbose output will be provided') + + parser.add_argument('--manifest', '-m', metavar='MANIFEST_PATH', type=Path, action='store', dest='manifest', + help='manifest file generated by epicli') + # offline mode rerun options: parser.add_argument('--rerun', action='store_true', dest='rerun', default=False, help=SUPPRESS) @@ -195,7 +209,44 @@ def __add_args(self, argv: List[str]): self.repos_backup_file = Path(args['repos_backup_file']) self.retries = args['retries'] self.is_log_file_enabled = False if args['no_logfile'] else True + self.dest_manifest = args['manifest'] or None + self.verbose_mode = args['verbose'] # offline mode self.rerun = args['rerun'] self.pyyaml_installed = args['pyyaml_installed'] + + def __print_parsed_manifest_data(self, output: Dict[str, Any]): + lines: List[str] = ['Manifest summary:'] + + lines.append('-' * self.__LINE_SIZE) + + lines.append('Components detected:') + for component in output['detected-components']: + lines.append(f'- {component}') + + lines.append('') + + lines.append('Features detected:') + for feature in output['detected-features']: + lines.append(f'- {feature}') + + lines.append('-' * self.__LINE_SIZE) + + logging.info('\n'.join(lines)) + + def read_manifest(self, requirements: Dict[str, Any]): + """ + Construct ManifestReader and parse only required data. + Not needed entries will be removed from the `requirements` + + :param requirements: parsed requirements which will be filtered based on the manifest output + """ + if not self.dest_manifest: + return + + mreader = ManifestReader(self.dest_manifest) + output = mreader.parse_manifest() + + if self.verbose_mode: + self.__print_parsed_manifest_data(output) diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/config/manifest_reader.py b/ansible/playbooks/roles/repository/files/download-requirements/src/config/manifest_reader.py new file mode 100644 index 0000000000..72ef3b2b5d --- /dev/null +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/config/manifest_reader.py @@ -0,0 +1,78 @@ +from pathlib import Path +from typing import Any, Callable, Dict, List, Set + +import yaml + +from src.error import CriticalError + + +def load_yaml_file_all(filename: Path) -> List[Any]: + try: + with open(filename, encoding="utf-8") as req_handler: + return list(yaml.safe_load_all(req_handler)) + except yaml.YAMLError as yaml_err: + raise CriticalError(f'Failed loading: `{yaml_err}`') from yaml_err + except Exception as err: + raise CriticalError(f'Failed loading: `{filename}`') from err + + +def load_yaml_file(filename: Path) -> Any: + return load_yaml_file_all(filename)[0] + + +class ManifestReader: + """ + Load the manifest file and call defined parser methods to process required documents. + Main running method is :func:`~manifest_reader.ManifestReader.parse_manifest` which returns formatted manifest output. + """ + + def __init__(self, dest_manifest: Path): + self.__dest_manifest = dest_manifest + self.__detected_components: Set = set() + self.__detected_features: Set = set() + + def __parse_cluster_info(self, cluster_doc: Dict): + """ + Parse `epiphany-cluster` document and extract only used components. + + :param cluster_doc: handler to a `epiphany-cluster` document + """ + components = cluster_doc['specification']['components'] + for component in components: + if components[component]['count'] > 0: + self.__detected_components.add(component) + + def __parse_feature_mappings_info(self, feature_mappings_doc: Dict): + """ + Parse `configuration/feature-mappings` document and extract only used features (based on `epiphany-cluster` doc). + + :param feature_mappings_doc: handler to a `configuration/feature-mappings` document + """ + mappings = feature_mappings_doc['specification']['mappings'] + for mapping in mappings.keys() & self.__detected_components: + for feature in mappings[mapping]: + self.__detected_features.add(feature) + + def parse_manifest(self) -> Dict[str, Any]: + """ + Load the manifest file, call parsers on required docs and return formatted output. + """ + parse_doc: Dict[str, Callable] = { + 'epiphany-cluster': self.__parse_cluster_info, + 'configuration/feature-mappings': self.__parse_feature_mappings_info + } + + parsed_docs: Set[str] = set() + for manifest_doc in load_yaml_file_all(self.__dest_manifest): + try: + kind: str = manifest_doc['kind'] + parse_doc[kind](manifest_doc) + parsed_docs.add(kind) + except KeyError: + pass + + if len(parsed_docs) != len(parse_doc.keys()): + raise CriticalError(f'ManifestReader - could not find documents: {parsed_docs ^ parse_doc.keys()}') + + return {'detected-components': sorted(list(self.__detected_components)), + 'detected-features': sorted(list(self.__detected_features))} diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/mode/base_mode.py b/ansible/playbooks/roles/repository/files/download-requirements/src/mode/base_mode.py index 980841243b..f011871435 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/mode/base_mode.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/mode/base_mode.py @@ -4,25 +4,14 @@ from pathlib import Path from typing import Any, Dict -import yaml - from src.command.toolchain import Toolchain, TOOLCHAINS from src.config.config import Config, OSArch +from src.config.manifest_reader import load_yaml_file from src.crypt import SHA_ALGORITHMS from src.downloader import Downloader from src.error import CriticalError, ChecksumMismatch -def load_yaml_file(filename: Path) -> Any: - try: - with open(filename, encoding="utf-8") as req_handler: - return yaml.safe_load(req_handler) - except yaml.YAMLError as yaml_err: - raise CriticalError(f'Failed loading: `{yaml_err}`') from yaml_err - except Exception as err: - raise CriticalError(f'Failed loading: `{filename}`') from err - - class BaseMode: """ An abstract class for running specific operations on target OS. @@ -35,6 +24,7 @@ def __init__(self, config: Config): self._repositories: Dict[str, Dict] = self.__parse_repositories() self._requirements: Dict[str, Any] = self.__parse_requirements() self._tools: Toolchain = TOOLCHAINS[self._cfg.os_type.os_family](self._cfg.retries) + self._cfg.read_manifest(self._requirements) def __parse_repositories(self) -> Dict[str, Dict]: """ diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt.py similarity index 97% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt.py index fa91bb4e44..086a9aeb32 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt.py @@ -1,6 +1,6 @@ from tests.mocks.command_run_mock import CommandRunMock -from src.command.apt import Apt +from src.command.debian.apt import Apt def test_interface_update(mocker): diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt_cache.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt_cache.py similarity index 93% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt_cache.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt_cache.py index 4a7008f476..359eb40238 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt_cache.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt_cache.py @@ -1,6 +1,6 @@ from tests.mocks.command_run_mock import CommandRunMock -from src.command.apt_cache import AptCache +from src.command.debian.apt_cache import AptCache def test_interface_get_package_dependencies(mocker): diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt_key.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt_key.py similarity index 88% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt_key.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt_key.py index 980398399d..d247466d68 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt_key.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt_key.py @@ -2,7 +2,7 @@ from tests.mocks.command_run_mock import CommandRunMock -from src.command.apt_key import AptKey +from src.command.debian.apt_key import AptKey def test_interface_add(mocker): diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf.py similarity index 97% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf.py index fc94fc92e8..46cfb92cd5 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf.py @@ -1,4 +1,4 @@ -from src.command.dnf import Dnf +from src.command.redhat.dnf import Dnf from tests.mocks.command_run_mock import CommandRunMock diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_config_manager.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_config_manager.py similarity index 93% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_config_manager.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_config_manager.py index 9835bebef9..ae9ab2dc47 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_config_manager.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_config_manager.py @@ -1,6 +1,6 @@ from tests.mocks.command_run_mock import CommandRunMock -from src.command.dnf_config_manager import DnfConfigManager +from src.command.redhat.dnf_config_manager import DnfConfigManager def test_interface_add_repo(mocker): diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_download.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_download.py similarity index 94% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_download.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_download.py index 4b689d6ffb..911aeee576 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_download.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_download.py @@ -2,7 +2,7 @@ from tests.mocks.command_run_mock import CommandRunMock -from src.command.dnf_download import DnfDownload +from src.command.redhat.dnf_download import DnfDownload def test_interface_download_packages(mocker): diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_repoquery.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_repoquery.py similarity index 96% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_repoquery.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_repoquery.py index 158509576d..44a9b94dc9 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_repoquery.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_repoquery.py @@ -1,6 +1,6 @@ from tests.mocks.command_run_mock import CommandRunMock -from src.command.dnf_repoquery import DnfRepoquery +from src.command.redhat.dnf_repoquery import DnfRepoquery def test_interface_query(mocker): diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_rpm.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_rpm.py similarity index 96% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_rpm.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_rpm.py index 57baf41eb7..1423889d44 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_rpm.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_rpm.py @@ -1,6 +1,6 @@ from tests.mocks.command_run_mock import CommandRunMock -from src.command.rpm import Rpm +from src.command.redhat.rpm import Rpm def test_interface_is_package_installed(mocker): diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py new file mode 100644 index 0000000000..081fc47b27 --- /dev/null +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py @@ -0,0 +1,30 @@ +import logging +from pathlib import Path + +import yaml + +from src.config.config import Config +from tests.data.config import EXPECTED_VERBOSE_OUTPUT +from tests.data.manifest_reader import INPUT_MANIFEST_FEATURE_MAPPINGS + + +def test_manifest_verbose_output(mocker, caplog): + ''' Check output produced when running download-requirements script with the `-v|--verbose` flag and with provided `-m|--manifest` ''' + + mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_MANIFEST_FEATURE_MAPPINGS)) + caplog.set_level(logging.INFO) + + # mock Config's init methods: + Config._Config__add_args = lambda *args: None + Config._Config__log_info_summary = lambda *args: None + + config = Config([]) + + # mock required config data: + config.dest_manifest = Path('/some/path') + config.verbose_mode = True + config.read_manifest({}) + + log_output = f'\n{"".join(caplog.messages)}\n' + + assert log_output == EXPECTED_VERBOSE_OUTPUT diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py new file mode 100644 index 0000000000..c4f04777d2 --- /dev/null +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py @@ -0,0 +1,13 @@ +from pathlib import Path + +import yaml + +from src.config.manifest_reader import ManifestReader +from tests.data.manifest_reader import EXPECTED_FEATURE_MAPPINGS, INPUT_MANIFEST_FEATURE_MAPPINGS + +def test_parse_manifest(mocker): + ''' Check manifest file parsing ''' + mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_MANIFEST_FEATURE_MAPPINGS)) + + mreader = ManifestReader(Path('/some/path')) + assert mreader.parse_manifest() == EXPECTED_FEATURE_MAPPINGS diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py new file mode 100644 index 0000000000..9275ba6aea --- /dev/null +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py @@ -0,0 +1,22 @@ +EXPECTED_VERBOSE_OUTPUT = """ +Manifest summary: +-------------------------------------------------- +Components detected: +- kafka +- monitoring +- repository + +Features detected: +- filebeat +- firewall +- grafana +- image-registry +- jmx-exporter +- kafka +- kafka-exporter +- node-exporter +- prometheus +- repository +- zookeeper +-------------------------------------------------- +""" diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/data/manifest_reader.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/data/manifest_reader.py new file mode 100644 index 0000000000..9e6f1da501 --- /dev/null +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/data/manifest_reader.py @@ -0,0 +1,142 @@ +INPUT_MANIFEST_FEATURE_MAPPINGS = """ +--- +kind: epiphany-cluster +title: Epiphany cluster Config +provider: any +name: default +specification: + name: new_cluster + admin_user: + name: operations + key_path: /shared/.ssh/epiphany-operations/id_rsa + components: + repository: + count: 1 + kubernetes_master: + count: 0 + kubernetes_node: + count: 0 + logging: + count: 0 + monitoring: + count: 1 + kafka: + count: 2 + postgresql: + count: 0 + load_balancer: + count: 0 + rabbitmq: + count: 0 + opendistro_for_elasticsearch: + count: 0 +version: 2.0.0dev +--- +kind: configuration/feature-mappings +title: Feature mapping to roles +name: default +specification: + mappings: + kafka: + - zookeeper + - jmx-exporter + - kafka + - kafka-exporter + - node-exporter + - filebeat + - firewall + rabbitmq: + - rabbitmq + - node-exporter + - filebeat + - firewall + logging: + - logging + - kibana + - node-exporter + - filebeat + - firewall + load_balancer: + - haproxy + - node-exporter + - filebeat + - firewall + monitoring: + - prometheus + - grafana + - node-exporter + - filebeat + - firewall + postgresql: + - postgresql + - postgres-exporter + - node-exporter + - filebeat + - firewall + custom: + - repository + - image-registry + - kubernetes-master + - node-exporter + - filebeat + - rabbitmq + - postgresql + - prometheus + - grafana + - node-exporter + - logging + - firewall + - rook + single_machine: + - repository + - image-registry + - kubernetes-master + - helm + - applications + - rabbitmq + - postgresql + - firewall + - rook + kubernetes_master: + - kubernetes-master + - helm + - applications + - rook + - node-exporter + - filebeat + - firewall + kubernetes_node: + - kubernetes-node + - node-exporter + - filebeat + - firewall + opendistro_for_elasticsearch: + - opendistro-for-elasticsearch + - node-exporter + - filebeat + - firewall + repository: + - repository + - image-registry + - firewall + - filebeat + - node-exporter +version: 2.0.0dev +provider: azure +""" + + +EXPECTED_FEATURE_MAPPINGS = { + 'detected-components': ['kafka', 'monitoring', 'repository'], + 'detected-features': ['filebeat', + 'firewall', + 'grafana', + 'image-registry', + 'jmx-exporter', + 'kafka', + 'kafka-exporter', + 'node-exporter', + 'prometheus', + 'repository', + 'zookeeper'] +} diff --git a/ansible/playbooks/roles/repository/tasks/copy-download-requirements.yml b/ansible/playbooks/roles/repository/tasks/copy-download-requirements.yml index d6cb17c4f8..e0db01cf76 100644 --- a/ansible/playbooks/roles/repository/tasks/copy-download-requirements.yml +++ b/ansible/playbooks/roles/repository/tasks/copy-download-requirements.yml @@ -55,6 +55,11 @@ dest: "{{ download_requirements_dir }}/{{ item }}" loop: "{{ _files }}" + - name: Copy the manifest file + synchronize: + src: "{{ inventory_dir }}/manifest.yml" + dest: "{{ download_requirements_dir }}/manifest.yml" + - name: Copy RedHat family specific download requirements file synchronize: src: "download-requirements/{{ _family_packages }}" diff --git a/ansible/playbooks/roles/repository/tasks/download-requirements.yml b/ansible/playbooks/roles/repository/tasks/download-requirements.yml index 534f23675f..9ca75265e4 100644 --- a/ansible/playbooks/roles/repository/tasks/download-requirements.yml +++ b/ansible/playbooks/roles/repository/tasks/download-requirements.yml @@ -10,8 +10,10 @@ "{{ download_requirements_script }}" \ /var/www/html/epirepo \ "{{ download_requirements_os_name }}" \ + --manifest "{{ download_requirements_manifest }}" \ + --no-logfile \ --repos-backup-file /var/tmp/enabled-system-repos.tar \ - --no-logfile |& + --verbose |& tee >(systemd-cat --identifier=download-requirements.py) args: executable: /bin/bash