From a4fe752ca4bb046520ce1a505045b1d73588497a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Bia=C5=82obrzecki?= Date: Tue, 10 May 2022 09:14:42 +0200 Subject: [PATCH] Optimize files downloading (#3116) --- .../requirements/x86_64/files.yml | 14 +++ .../src/config/config.py | 43 ++++--- .../tests/config/test_config.py | 73 ++++++++---- .../tests/data/config.py | 109 ++++++++++++++++-- 4 files changed, 197 insertions(+), 42 deletions(-) diff --git a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/files.yml b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/files.yml index ec2f266d7a..dee75663e3 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/files.yml +++ b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/files.yml @@ -3,44 +3,58 @@ files: # --- Exporters --- 'https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-amd64.tar.gz': sha256: ffda682e82daede726da8719257a088f8e23dcaa4e2ac8b2b2748a129aea85f0 + deps: kafka-exporter 'https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar': sha256: 0ddc6834f854c03d5795305193c1d33132a24fbd406b4b52828602f5bc30777e + deps: prometheus 'https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz': sha256: 68f3802c2dd3980667e4ba65ea2e1fb03f4a4ba026cca375f15a0390ff850949 + deps: node-exporter 'https://github.com/prometheus-community/postgres_exporter/releases/download/v0.10.0/postgres_exporter-0.10.0.linux-amd64.tar.gz': sha256: 1d1a008c5e29673b404a9ce119b7516fa59974aeda2f47d4a0446d102abce8a1 + deps: postgres-exporter # --- Misc --- 'https://archive.apache.org/dist/kafka/2.8.1/kafka_2.12-2.8.1.tgz': sha256: 175a4134efc569a586d58916cd16ce70f868b13dea2b5a3d12a67b1395d59f98 + deps: kafka 'https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz': sha256: c35ed6786d59b73920243f1a324d24c2ddfafb379041d7a350cc9a341c52caf3 + deps: zookeeper 'https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz': sha256: 77793c4d9bb92be98f7525f8bc50cb8adb8c5de2e944d5500e90ab13918771fc + deps: alertmanager 'https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz': sha256: 7852dc11cfaa039577c1804fe6f082a07c5eb06be50babcffe29214aedf318b3 + deps: prometheus 'https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz': sha256: 4c3fd562e64005786ac8f18e7334054a24da34ec04bbd769c206b03b8ed6e457 + deps: helm 'https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz': sha256: b876c20c9d318d77a39c0c2e095897b2bb1cd100c7859643f8c7c8b0fc6d5961 + deps: default # --- Helm charts --- 'https://charts.bitnami.com/bitnami/node-exporter-2.3.17.tgz': sha256: ec586fabb775a4f05510386899cf348391523c89ff5a1d4097b0592e675ade7f + deps: node-exporter 'https://helm.elastic.co/helm/filebeat/filebeat-7.12.1.tgz': sha256: 5838058fe06372390dc335900a7707109cc7287a84164ca245d395af1f9c0a79 + deps: filebeat 'https://charts.rook.io/release/rook-ceph-v1.8.8.tgz': sha256: f67e474dedffd4004f3a0b7b40112694a7f1c2b1a0048b03b3083d0a01e86b14 + deps: rook 'https://charts.rook.io/release/rook-ceph-cluster-v1.8.8.tgz': sha256: df4e1f2125af41fb84c72e4d12aa0cb859dddd4f37b3d5979981bd092040bd16 + deps: rook diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py b/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py index b1bd8228ee..e4eec87df1 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py @@ -216,32 +216,49 @@ def __add_args(self, argv: List[str]): self.rerun = args['rerun'] self.pyyaml_installed = args['pyyaml_installed'] - def __print_parsed_manifest_data(self, requirements: Dict[str, Any], output: Dict[str, Any]): + def __print_parsed_manifest_data(self, requirements: Dict[str, Any], manifest: Dict[str, Any]): lines: List[str] = ['Manifest summary:'] lines.append('-' * self.__LINE_SIZE) lines.append('Components detected:') - for component in output['detected-components']: + for component in manifest['detected-components']: lines.append(f'- {component}') lines.append('') lines.append('Features detected:') - for feature in output['detected-features']: + for feature in manifest['detected-features']: lines.append(f'- {feature}') - dashboards = requirements['grafana-dashboards'] - if dashboards: - lines.append('') - lines.append('Dashboards to download:') - for dashboard in dashboards: - lines.append(f'- {dashboard}') + for reqs in [('files', 'Files'), ('grafana-dashboards', 'Dashboards')]: + reqs_to_download = sorted(requirements[reqs[0]]) + if reqs_to_download: + lines.append('') + lines.append(f'{reqs[1]} to download:') + for req_to_download in reqs_to_download: + lines.append(f'- {req_to_download}') lines.append('-' * self.__LINE_SIZE) logging.info('\n'.join(lines)) + def __filter_manifest(self, requirements: Dict[str, Any], manifest: Dict[str, Any]): + """ + """ + if 'grafana' not in manifest['detected-features']: + requirements['grafana-dashboards'] = [] + + files = requirements['files'] + files_to_exclude: List[str] = [] + for file in files: + deps = files[file]['deps'] + if deps not in manifest['detected-features'] and deps != 'default': + files_to_exclude.append(file) + + if files_to_exclude: + requirements['files'] = {url: data for url, data in files.items() if url not in files_to_exclude} + def read_manifest(self, requirements: Dict[str, Any]): """ Construct ManifestReader and parse only required data. @@ -253,10 +270,8 @@ def read_manifest(self, requirements: Dict[str, Any]): return mreader = ManifestReader(self.dest_manifest) - output = mreader.parse_manifest() - - if 'grafana' not in output['detected-features']: - requirements['grafana-dashboards'] = [] + manifest = mreader.parse_manifest() + self.__filter_manifest(requirements, manifest) if self.verbose_mode: - self.__print_parsed_manifest_data(requirements, output) + self.__print_parsed_manifest_data(requirements, manifest) diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py index fda58df0dc..d36aa474ff 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py @@ -1,34 +1,67 @@ -import logging from pathlib import Path +import logging import pytest import yaml from src.config.config import Config -from tests.data.config import EXPECTED_VERBOSE_OUTPUT, EXPECTED_VERBOSE_DASHBOARD_OUTPUT, DASHBOARD_REQUIREMENTS -from tests.data.manifest_reader import INPUT_MANIFEST_FEATURE_MAPPINGS, INPUT_MANIFEST_WITH_DASHBOARDS +from tests.data.config import (DASHBOARD_REQUIREMENTS, + EXPECTED_VERBOSE_OUTPUT, + EXPECTED_VERBOSE_DASHBOARD_OUTPUT, + EXPECTED_VERBOSE_FILE_OUTPUT, + FILE_REQUIREMENTS) +from tests.data.manifest_reader import (INPUT_MANIFEST_FEATURE_MAPPINGS, + INPUT_MANIFEST_WITH_DASHBOARDS) + + +@pytest.fixture(scope='class') +def default_manifest(request): + """ + Generate default requirements for each test + """ + request.cls.MANIFEST = { + 'files': '', + 'grafana-dashboards': '' + } + + +@pytest.mark.usefixtures('default_manifest') +class TestConfig: + @pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC, REQUIREMENTS', + [(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_VERBOSE_OUTPUT, DASHBOARD_REQUIREMENTS), + (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_VERBOSE_DASHBOARD_OUTPUT, DASHBOARD_REQUIREMENTS), + (INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_VERBOSE_FILE_OUTPUT, FILE_REQUIREMENTS) + ]) + def test_manifest_verbose_output(self, INPUT_DOC: str, + EXPECTED_OUTPUT_DOC: str, + REQUIREMENTS: str, + mocker, caplog): + """ + Check output produced when running download-requirements script with the `-v|--verbose` flag and with provided `-m|--manifest` + + :param INPUT_DOC: yaml doc which will be parsed by the ManifestReader + :param EXPECTED_OUTPUT_DOC: expected output to be printed by the `Config` class, then tested against the parsed `INPUT_DOC` + :param REQUIREMENTS: yaml doc containing requirements passed to `Config`'s read_manifest() + """ + mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_DOC)) + caplog.set_level(logging.INFO) -@pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC', - [(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_VERBOSE_OUTPUT), - (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_VERBOSE_DASHBOARD_OUTPUT)]) -def test_manifest_verbose_output(INPUT_DOC, EXPECTED_OUTPUT_DOC, mocker, caplog): - ''' Check output produced when running download-requirements script with the `-v|--verbose` flag and with provided `-m|--manifest` ''' + # mock Config's init methods: + Config._Config__add_args = lambda *args: None + Config._Config__log_info_summary = lambda *args: None - mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_DOC)) - caplog.set_level(logging.INFO) + config = Config([]) - # mock Config's init methods: - Config._Config__add_args = lambda *args: None - Config._Config__log_info_summary = lambda *args: None + # mock required config data: + config.dest_manifest = Path('/some/path') + config.verbose_mode = True - config = Config([]) + req_key, doc = tuple(yaml.safe_load(REQUIREMENTS).items())[0] - # mock required config data: - config.dest_manifest = Path('/some/path') - config.verbose_mode = True - config.read_manifest(yaml.safe_load(DASHBOARD_REQUIREMENTS)) + self.MANIFEST[req_key] = doc + config.read_manifest(self.MANIFEST) - log_output = f'\n{"".join(caplog.messages)}\n' + log_output = f'\n{"".join(caplog.messages)}\n' - assert log_output == EXPECTED_OUTPUT_DOC + assert log_output == EXPECTED_OUTPUT_DOC diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py index 581c68dbe4..041d83268b 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py @@ -1,3 +1,66 @@ +FILE_REQUIREMENTS = """ +files: + # --- Exporters --- + 'https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-amd64.tar.gz': + sha256: ffda682e82daede726da8719257a088f8e23dcaa4e2ac8b2b2748a129aea85f0 + deps: kafka-exporter + + 'https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar': + sha256: 0ddc6834f854c03d5795305193c1d33132a24fbd406b4b52828602f5bc30777e + deps: prometheus + + 'https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz': + sha256: 68f3802c2dd3980667e4ba65ea2e1fb03f4a4ba026cca375f15a0390ff850949 + deps: node-exporter + + 'https://github.com/prometheus-community/postgres_exporter/releases/download/v0.10.0/postgres_exporter-0.10.0.linux-amd64.tar.gz': + sha256: 1d1a008c5e29673b404a9ce119b7516fa59974aeda2f47d4a0446d102abce8a1 + deps: postgres-exporter + + # --- Misc --- + 'https://archive.apache.org/dist/kafka/2.8.1/kafka_2.12-2.8.1.tgz': + sha256: 175a4134efc569a586d58916cd16ce70f868b13dea2b5a3d12a67b1395d59f98 + deps: kafka + + 'https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz': + sha256: c35ed6786d59b73920243f1a324d24c2ddfafb379041d7a350cc9a341c52caf3 + deps: zookeeper + + 'https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz': + sha256: 77793c4d9bb92be98f7525f8bc50cb8adb8c5de2e944d5500e90ab13918771fc + deps: alertmanager + + 'https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz': + sha256: 7852dc11cfaa039577c1804fe6f082a07c5eb06be50babcffe29214aedf318b3 + deps: prometheus + + 'https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz': + sha256: 4c3fd562e64005786ac8f18e7334054a24da34ec04bbd769c206b03b8ed6e457 + deps: helm + + 'https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz': + sha256: b876c20c9d318d77a39c0c2e095897b2bb1cd100c7859643f8c7c8b0fc6d5961 + deps: default + + # --- Helm charts --- + 'https://charts.bitnami.com/bitnami/node-exporter-2.3.17.tgz': + sha256: ec586fabb775a4f05510386899cf348391523c89ff5a1d4097b0592e675ade7f + deps: node-exporter + + 'https://helm.elastic.co/helm/filebeat/filebeat-7.12.1.tgz': + sha256: 5838058fe06372390dc335900a7707109cc7287a84164ca245d395af1f9c0a79 + deps: filebeat + + 'https://charts.rook.io/release/rook-ceph-v1.8.8.tgz': + sha256: f67e474dedffd4004f3a0b7b40112694a7f1c2b1a0048b03b3083d0a01e86b14 + deps: rook + + 'https://charts.rook.io/release/rook-ceph-cluster-v1.8.8.tgz': + sha256: df4e1f2125af41fb84c72e4d12aa0cb859dddd4f37b3d5979981bd092040bd16 + deps: rook +""" + + DASHBOARD_REQUIREMENTS = """ grafana-dashboards: grafana_dashboard_7249: @@ -80,18 +143,48 @@ - repository Dashboards to download: -- grafana_dashboard_7249 -- grafana_dashboard_315 +- grafana_dashboard_10991 - grafana_dashboard_11074 +- grafana_dashboard_179 +- grafana_dashboard_1860 +- grafana_dashboard_315 - grafana_dashboard_405 -- grafana_dashboard_455 -- grafana_dashboard_9628 - grafana_dashboard_4279 -- grafana_dashboard_1860 +- grafana_dashboard_455 +- grafana_dashboard_6663 +- grafana_dashboard_7249 - grafana_dashboard_7589 - grafana_dashboard_789 -- grafana_dashboard_179 -- grafana_dashboard_6663 -- grafana_dashboard_10991 +- grafana_dashboard_9628 +-------------------------------------------------- +""" + + +EXPECTED_VERBOSE_FILE_OUTPUT = """ +Manifest summary: +-------------------------------------------------- +Components detected: +- kafka +- repository + +Features detected: +- filebeat +- firewall +- image-registry +- jmx-exporter +- kafka +- kafka-exporter +- node-exporter +- repository +- zookeeper + +Files to download: +- https://archive.apache.org/dist/kafka/2.8.1/kafka_2.12-2.8.1.tgz +- https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz +- https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz +- https://charts.bitnami.com/bitnami/node-exporter-2.3.17.tgz +- https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-amd64.tar.gz +- https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz +- https://helm.elastic.co/helm/filebeat/filebeat-7.12.1.tgz -------------------------------------------------- """