Skip to content

Commit

Permalink
Optimize files downloading (hitachienergy#3116)
Browse files Browse the repository at this point in the history
  • Loading branch information
sbbroot committed May 20, 2022
1 parent fa27451 commit d371e62
Show file tree
Hide file tree
Showing 4 changed files with 197 additions and 42 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -3,44 +3,58 @@ files:
# --- Exporters ---
'https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-amd64.tar.gz':
sha256: ffda682e82daede726da8719257a088f8e23dcaa4e2ac8b2b2748a129aea85f0
deps: kafka-exporter

'https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar':
sha256: 0ddc6834f854c03d5795305193c1d33132a24fbd406b4b52828602f5bc30777e
deps: prometheus

'https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz':
sha256: 68f3802c2dd3980667e4ba65ea2e1fb03f4a4ba026cca375f15a0390ff850949
deps: node-exporter

'https://github.com/prometheus-community/postgres_exporter/releases/download/v0.10.0/postgres_exporter-0.10.0.linux-amd64.tar.gz':
sha256: 1d1a008c5e29673b404a9ce119b7516fa59974aeda2f47d4a0446d102abce8a1
deps: postgres-exporter

# --- Misc ---
'https://archive.apache.org/dist/kafka/2.8.1/kafka_2.12-2.8.1.tgz':
sha256: 175a4134efc569a586d58916cd16ce70f868b13dea2b5a3d12a67b1395d59f98
deps: kafka

'https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz':
sha256: c35ed6786d59b73920243f1a324d24c2ddfafb379041d7a350cc9a341c52caf3
deps: zookeeper

'https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz':
sha256: 77793c4d9bb92be98f7525f8bc50cb8adb8c5de2e944d5500e90ab13918771fc
deps: alertmanager

'https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz':
sha256: 7852dc11cfaa039577c1804fe6f082a07c5eb06be50babcffe29214aedf318b3
deps: prometheus

'https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz':
sha256: 4c3fd562e64005786ac8f18e7334054a24da34ec04bbd769c206b03b8ed6e457
deps: helm

'https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz':
sha256: b876c20c9d318d77a39c0c2e095897b2bb1cd100c7859643f8c7c8b0fc6d5961
deps: default

# --- Helm charts ---
'https://charts.bitnami.com/bitnami/node-exporter-2.3.17.tgz':
sha256: ec586fabb775a4f05510386899cf348391523c89ff5a1d4097b0592e675ade7f
deps: node-exporter

'https://helm.elastic.co/helm/filebeat/filebeat-7.12.1.tgz':
sha256: 5838058fe06372390dc335900a7707109cc7287a84164ca245d395af1f9c0a79
deps: filebeat

'https://charts.rook.io/release/rook-ceph-v1.8.8.tgz':
sha256: f67e474dedffd4004f3a0b7b40112694a7f1c2b1a0048b03b3083d0a01e86b14
deps: rook

'https://charts.rook.io/release/rook-ceph-cluster-v1.8.8.tgz':
sha256: df4e1f2125af41fb84c72e4d12aa0cb859dddd4f37b3d5979981bd092040bd16
deps: rook
Original file line number Diff line number Diff line change
Expand Up @@ -216,32 +216,49 @@ def __add_args(self, argv: List[str]):
self.rerun = args['rerun']
self.pyyaml_installed = args['pyyaml_installed']

def __print_parsed_manifest_data(self, requirements: Dict[str, Any], output: Dict[str, Any]):
def __print_parsed_manifest_data(self, requirements: Dict[str, Any], manifest: Dict[str, Any]):
lines: List[str] = ['Manifest summary:']

lines.append('-' * self.__LINE_SIZE)

lines.append('Components detected:')
for component in output['detected-components']:
for component in manifest['detected-components']:
lines.append(f'- {component}')

lines.append('')

lines.append('Features detected:')
for feature in output['detected-features']:
for feature in manifest['detected-features']:
lines.append(f'- {feature}')

dashboards = requirements['grafana-dashboards']
if dashboards:
lines.append('')
lines.append('Dashboards to download:')
for dashboard in dashboards:
lines.append(f'- {dashboard}')
for reqs in [('files', 'Files'), ('grafana-dashboards', 'Dashboards')]:
reqs_to_download = sorted(requirements[reqs[0]])
if reqs_to_download:
lines.append('')
lines.append(f'{reqs[1]} to download:')
for req_to_download in reqs_to_download:
lines.append(f'- {req_to_download}')

lines.append('-' * self.__LINE_SIZE)

logging.info('\n'.join(lines))

def __filter_manifest(self, requirements: Dict[str, Any], manifest: Dict[str, Any]):
"""
"""
if 'grafana' not in manifest['detected-features']:
requirements['grafana-dashboards'] = []

files = requirements['files']
files_to_exclude: List[str] = []
for file in files:
deps = files[file]['deps']
if deps not in manifest['detected-features'] and deps != 'default':
files_to_exclude.append(file)

if files_to_exclude:
requirements['files'] = {url: data for url, data in files.items() if url not in files_to_exclude}

def read_manifest(self, requirements: Dict[str, Any]):
"""
Construct ManifestReader and parse only required data.
Expand All @@ -253,10 +270,8 @@ def read_manifest(self, requirements: Dict[str, Any]):
return

mreader = ManifestReader(self.dest_manifest)
output = mreader.parse_manifest()

if 'grafana' not in output['detected-features']:
requirements['grafana-dashboards'] = []
manifest = mreader.parse_manifest()
self.__filter_manifest(requirements, manifest)

if self.verbose_mode:
self.__print_parsed_manifest_data(requirements, output)
self.__print_parsed_manifest_data(requirements, manifest)
Original file line number Diff line number Diff line change
@@ -1,34 +1,67 @@
import logging
from pathlib import Path
import logging

import pytest
import yaml

from src.config.config import Config
from tests.data.config import EXPECTED_VERBOSE_OUTPUT, EXPECTED_VERBOSE_DASHBOARD_OUTPUT, DASHBOARD_REQUIREMENTS
from tests.data.manifest_reader import INPUT_MANIFEST_FEATURE_MAPPINGS, INPUT_MANIFEST_WITH_DASHBOARDS
from tests.data.config import (DASHBOARD_REQUIREMENTS,
EXPECTED_VERBOSE_OUTPUT,
EXPECTED_VERBOSE_DASHBOARD_OUTPUT,
EXPECTED_VERBOSE_FILE_OUTPUT,
FILE_REQUIREMENTS)
from tests.data.manifest_reader import (INPUT_MANIFEST_FEATURE_MAPPINGS,
INPUT_MANIFEST_WITH_DASHBOARDS)


@pytest.fixture(scope='class')
def default_manifest(request):
"""
Generate default requirements for each test
"""
request.cls.MANIFEST = {
'files': '',
'grafana-dashboards': ''
}


@pytest.mark.usefixtures('default_manifest')
class TestConfig:
@pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC, REQUIREMENTS',
[(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_VERBOSE_OUTPUT, DASHBOARD_REQUIREMENTS),
(INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_VERBOSE_DASHBOARD_OUTPUT, DASHBOARD_REQUIREMENTS),
(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_VERBOSE_FILE_OUTPUT, FILE_REQUIREMENTS)
])
def test_manifest_verbose_output(self, INPUT_DOC: str,
EXPECTED_OUTPUT_DOC: str,
REQUIREMENTS: str,
mocker, caplog):
"""
Check output produced when running download-requirements script with the `-v|--verbose` flag and with provided `-m|--manifest`
:param INPUT_DOC: yaml doc which will be parsed by the ManifestReader
:param EXPECTED_OUTPUT_DOC: expected output to be printed by the `Config` class, then tested against the parsed `INPUT_DOC`
:param REQUIREMENTS: yaml doc containing requirements passed to `Config`'s read_manifest()
"""

mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_DOC))
caplog.set_level(logging.INFO)

@pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC',
[(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_VERBOSE_OUTPUT),
(INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_VERBOSE_DASHBOARD_OUTPUT)])
def test_manifest_verbose_output(INPUT_DOC, EXPECTED_OUTPUT_DOC, mocker, caplog):
''' Check output produced when running download-requirements script with the `-v|--verbose` flag and with provided `-m|--manifest` '''
# mock Config's init methods:
Config._Config__add_args = lambda *args: None
Config._Config__log_info_summary = lambda *args: None

mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_DOC))
caplog.set_level(logging.INFO)
config = Config([])

# mock Config's init methods:
Config._Config__add_args = lambda *args: None
Config._Config__log_info_summary = lambda *args: None
# mock required config data:
config.dest_manifest = Path('/some/path')
config.verbose_mode = True

config = Config([])
req_key, doc = tuple(yaml.safe_load(REQUIREMENTS).items())[0]

# mock required config data:
config.dest_manifest = Path('/some/path')
config.verbose_mode = True
config.read_manifest(yaml.safe_load(DASHBOARD_REQUIREMENTS))
self.MANIFEST[req_key] = doc
config.read_manifest(self.MANIFEST)

log_output = f'\n{"".join(caplog.messages)}\n'
log_output = f'\n{"".join(caplog.messages)}\n'

assert log_output == EXPECTED_OUTPUT_DOC
assert log_output == EXPECTED_OUTPUT_DOC
Original file line number Diff line number Diff line change
@@ -1,3 +1,66 @@
FILE_REQUIREMENTS = """
files:
# --- Exporters ---
'https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-amd64.tar.gz':
sha256: ffda682e82daede726da8719257a088f8e23dcaa4e2ac8b2b2748a129aea85f0
deps: kafka-exporter
'https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar':
sha256: 0ddc6834f854c03d5795305193c1d33132a24fbd406b4b52828602f5bc30777e
deps: prometheus
'https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz':
sha256: 68f3802c2dd3980667e4ba65ea2e1fb03f4a4ba026cca375f15a0390ff850949
deps: node-exporter
'https://github.com/prometheus-community/postgres_exporter/releases/download/v0.10.0/postgres_exporter-0.10.0.linux-amd64.tar.gz':
sha256: 1d1a008c5e29673b404a9ce119b7516fa59974aeda2f47d4a0446d102abce8a1
deps: postgres-exporter
# --- Misc ---
'https://archive.apache.org/dist/kafka/2.8.1/kafka_2.12-2.8.1.tgz':
sha256: 175a4134efc569a586d58916cd16ce70f868b13dea2b5a3d12a67b1395d59f98
deps: kafka
'https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz':
sha256: c35ed6786d59b73920243f1a324d24c2ddfafb379041d7a350cc9a341c52caf3
deps: zookeeper
'https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz':
sha256: 77793c4d9bb92be98f7525f8bc50cb8adb8c5de2e944d5500e90ab13918771fc
deps: alertmanager
'https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz':
sha256: 7852dc11cfaa039577c1804fe6f082a07c5eb06be50babcffe29214aedf318b3
deps: prometheus
'https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz':
sha256: 4c3fd562e64005786ac8f18e7334054a24da34ec04bbd769c206b03b8ed6e457
deps: helm
'https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz':
sha256: b876c20c9d318d77a39c0c2e095897b2bb1cd100c7859643f8c7c8b0fc6d5961
deps: default
# --- Helm charts ---
'https://charts.bitnami.com/bitnami/node-exporter-2.3.17.tgz':
sha256: ec586fabb775a4f05510386899cf348391523c89ff5a1d4097b0592e675ade7f
deps: node-exporter
'https://helm.elastic.co/helm/filebeat/filebeat-7.12.1.tgz':
sha256: 5838058fe06372390dc335900a7707109cc7287a84164ca245d395af1f9c0a79
deps: filebeat
'https://charts.rook.io/release/rook-ceph-v1.8.8.tgz':
sha256: f67e474dedffd4004f3a0b7b40112694a7f1c2b1a0048b03b3083d0a01e86b14
deps: rook
'https://charts.rook.io/release/rook-ceph-cluster-v1.8.8.tgz':
sha256: df4e1f2125af41fb84c72e4d12aa0cb859dddd4f37b3d5979981bd092040bd16
deps: rook
"""


DASHBOARD_REQUIREMENTS = """
grafana-dashboards:
grafana_dashboard_7249:
Expand Down Expand Up @@ -80,18 +143,48 @@
- repository
Dashboards to download:
- grafana_dashboard_7249
- grafana_dashboard_315
- grafana_dashboard_10991
- grafana_dashboard_11074
- grafana_dashboard_179
- grafana_dashboard_1860
- grafana_dashboard_315
- grafana_dashboard_405
- grafana_dashboard_455
- grafana_dashboard_9628
- grafana_dashboard_4279
- grafana_dashboard_1860
- grafana_dashboard_455
- grafana_dashboard_6663
- grafana_dashboard_7249
- grafana_dashboard_7589
- grafana_dashboard_789
- grafana_dashboard_179
- grafana_dashboard_6663
- grafana_dashboard_10991
- grafana_dashboard_9628
--------------------------------------------------
"""


EXPECTED_VERBOSE_FILE_OUTPUT = """
Manifest summary:
--------------------------------------------------
Components detected:
- kafka
- repository
Features detected:
- filebeat
- firewall
- image-registry
- jmx-exporter
- kafka
- kafka-exporter
- node-exporter
- repository
- zookeeper
Files to download:
- https://archive.apache.org/dist/kafka/2.8.1/kafka_2.12-2.8.1.tgz
- https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz
- https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz
- https://charts.bitnami.com/bitnami/node-exporter-2.3.17.tgz
- https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-amd64.tar.gz
- https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz
- https://helm.elastic.co/helm/filebeat/filebeat-7.12.1.tgz
--------------------------------------------------
"""

0 comments on commit d371e62

Please sign in to comment.